Android video thumbnail: Generate thumbnails for vp8, vp9 videos.

This CL does the following:

1. Use VpxVideoDecoder in utility process to do software decoding for
vp8, vp9 video files.

2. Simplify the class to render media::VideoFrame into bitmap, since
PaintCanvasVideoRenderer works for all kind of VideoFrames.

3. Do not retrieve poster image from metadata. The mime type of the
poster image is inferred from mime sniffer, and needs to be
processed in sandbox process.


Bug: 826021
Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel
Change-Id: I91c9cf72a75be4bc3a95203a40633647e079a40a
Reviewed-on: https://chromium-review.googlesource.com/1234859
Commit-Queue: Xing Liu <xingliu@chromium.org>
Reviewed-by: David Trainor <dtrainor@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Tommy Li <tommycli@chromium.org>
Cr-Commit-Position: refs/heads/master@{#595523}

chrome/android/BUILD.gn

@@ -728,6 +728,8 @@ android_library("chrome_test_java") {
     "//content/test/data/media/video-player.html",
     "//content/test/data/media/webrtc_test_utilities.js",
     "//media/test/data/bear.mp4",
+    "//media/test/data/bear-vp8-webvtt.webm",
+    "//media/test/data/bear-vp8a.webm",
     "//media/test/data/sfx.mp3",
   ]
 }

chrome/android/javatests/src/org/chromium/chrome/browser/download/DownloadMediaParserTest.java

@@ -44,7 +44,7 @@ public class DownloadMediaParserTest {
     /**
      * Wraps result from download media parser.
      */
-    public static class MediaParseResult {
+    public static class MediaParserResult {
         public boolean done;
         public DownloadMediaData mediaData;
     }
@@ -54,21 +54,16 @@ public class DownloadMediaParserTest {
         mTestRule.loadNativeLibraryAndInitBrowserProcess();
     }
 
-    @Test
-    @LargeTest
-    @Feature({"Download"})
-    @RetryOnFailure
-    public void testParseAudioMetatadata() throws InterruptedException {
-        String filePath = UrlUtils.getIsolatedTestRoot() + "/media/test/data/sfx.mp3";
-        File audioFile = new File(filePath);
-        Assert.assertTrue(audioFile.exists());
+    private MediaParserResult parseMediaFile(String filePath, String mimeType) {
+        File mediaFile = new File(filePath);
+        Assert.assertTrue(mediaFile.exists());
         boolean done = false;
-        MediaParseResult result = new MediaParseResult();
+        MediaParserResult result = new MediaParserResult();
 
         // The native DownloadMediaParser needs to be created on UI thread.
         ThreadUtils.runOnUiThreadBlocking(() -> {
             DownloadMediaParserBridge parser = new DownloadMediaParserBridge(
-                    "audio/mp3", filePath, audioFile.length(), (DownloadMediaData mediaData) -> {
+                    mimeType, filePath, mediaFile.length(), (DownloadMediaData mediaData) -> {
                         result.mediaData = mediaData;
                         result.done = true;
                     });
@@ -81,7 +76,20 @@ public class DownloadMediaParserTest {
                 return result.done;
             }
         }, MAX_MEDIA_PARSER_POLL_TIME_MS, MEDIA_PARSER_POLL_INTERVAL_MS);
+        return result;
+    }
 
+    @Test
+    @LargeTest
+    @Feature({"Download"})
+    @RetryOnFailure
+    /**
+     * Verify that the metadata from audio file can be retrieved correctly.
+     * @throws InterruptedException
+     */
+    public void testParseAudioMetatadata() throws InterruptedException {
+        String filePath = UrlUtils.getIsolatedTestRoot() + "/media/test/data/sfx.mp3";
+        MediaParserResult result = parseMediaFile(filePath, "audio/mp3");
         Assert.assertTrue("Failed to parse audio metadata.", result.mediaData != null);
     }
 
@@ -89,30 +97,50 @@ public class DownloadMediaParserTest {
     @LargeTest
     @Feature({"Download"})
     @RetryOnFailure
-    public void testParseVideoMetatadataThumbnail() throws InterruptedException {
+    /**
+     * Verify metadata and thumbnail can be retrieved correctly from h264 video file.
+     * @throws InterruptedException
+     */
+    public void testParseVideoH264() throws InterruptedException {
         String filePath = UrlUtils.getIsolatedTestRoot() + "/media/test/data/bear.mp4";
-        File videoFile = new File(filePath);
-        Assert.assertTrue(videoFile.exists());
-        boolean done = false;
-        MediaParseResult result = new MediaParseResult();
-
-        // The native DownloadMediaParser needs to be created on UI thread.
-        ThreadUtils.runOnUiThreadBlocking(() -> {
-            DownloadMediaParserBridge parser = new DownloadMediaParserBridge(
-                    "video/mp4", filePath, videoFile.length(), (DownloadMediaData mediaData) -> {
-                        result.mediaData = mediaData;
-                        result.done = true;
-                    });
-            parser.start();
-        });
+        MediaParserResult result = parseMediaFile(filePath, "video/mp4");
+        Assert.assertTrue("Failed to parse video file.", result.mediaData != null);
+        Assert.assertTrue(
+                "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getWidth() > 0);
+        Assert.assertTrue(
+                "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getHeight() > 0);
+    }
 
-        CriteriaHelper.pollUiThread(new Criteria() {
-            @Override
-            public boolean isSatisfied() {
-                return result.done;
-            }
-        }, MAX_MEDIA_PARSER_POLL_TIME_MS, MEDIA_PARSER_POLL_INTERVAL_MS);
+    @Test
+    @LargeTest
+    @Feature({"Download"})
+    @RetryOnFailure
+    /**
+     * Verify metadata and thumbnail can be retrieved correctly from vp8 video file.
+     * @throws InterruptedException
+     */
+    public void testParseVideoThumbnailVp8() throws InterruptedException {
+        String filePath = UrlUtils.getIsolatedTestRoot() + "/media/test/data/bear-vp8-webvtt.webm";
+        MediaParserResult result = parseMediaFile(filePath, "video/webm");
+        Assert.assertTrue("Failed to parse video file.", result.mediaData != null);
+        Assert.assertTrue(
+                "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getWidth() > 0);
+        Assert.assertTrue(
+                "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getHeight() > 0);
+    }
 
+    @Test
+    @LargeTest
+    @Feature({"Download"})
+    @RetryOnFailure
+    /**
+     * Verify metadata and thumbnail can be retrieved correctly from vp8 video file with alpha
+     * plane.
+     * @throws InterruptedException
+     */
+    public void testParseVideoThumbnailVp8WithAlphaPlane() throws InterruptedException {
+        String filePath = UrlUtils.getIsolatedTestRoot() + "/media/test/data/bear-vp8a.webm";
+        MediaParserResult result = parseMediaFile(filePath, "video/webm");
         Assert.assertTrue("Failed to parse video file.", result.mediaData != null);
         Assert.assertTrue(
                 "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getWidth() > 0);

chrome/browser/BUILD.gn

@@ -2127,8 +2127,6 @@ jumbo_split_static_library("browser") {
       "android/download/service/download_background_task.cc",
       "android/download/service/download_task_scheduler.cc",
       "android/download/service/download_task_scheduler.h",
-      "android/download/video_frame_thumbnail_converter.cc",
-      "android/download/video_frame_thumbnail_converter.h",
       "android/explore_sites/catalog.cc",
       "android/explore_sites/catalog.h",
       "android/explore_sites/explore_sites_bridge.cc",

chrome/browser/android/download/download_media_parser.cc

@@ -9,15 +9,17 @@
 #include "base/numerics/safe_conversions.h"
 #include "base/task/post_task.h"
 #include "base/task/task_traits.h"
+#include "cc/paint/skia_paint_canvas.h"
 #include "chrome/browser/android/download/local_media_data_source_factory.h"
-#include "chrome/browser/android/download/video_frame_thumbnail_converter.h"
 #include "content/public/browser/android/gpu_video_accelerator_factories_provider.h"
 #include "content/public/common/service_manager_connection.h"
 #include "media/base/overlay_info.h"
+#include "media/base/video_thumbnail_decoder.h"
 #include "media/mojo/clients/mojo_video_decoder.h"
 #include "media/mojo/interfaces/constants.mojom.h"
 #include "media/mojo/interfaces/media_service.mojom.h"
 #include "media/mojo/services/media_interface_provider.h"
+#include "media/renderers/paint_canvas_video_renderer.h"
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "services/service_manager/public/cpp/connector.h"
 #include "services/ws/public/cpp/gpu/context_provider_command_buffer.h"
@@ -76,7 +78,7 @@ void DownloadMediaParser::OnMediaParserCreated() {
                                        weak_factory_.GetWeakPtr()));
 
   media_parser()->ParseMediaMetadata(
-      mime_type_, size_, true /* get_attached_images */, std::move(source_ptr),
+      mime_type_, size_, false /* get_attached_images */, std::move(source_ptr),
       base::BindOnce(&DownloadMediaParser::OnMediaMetadataParsed,
                      weak_factory_.GetWeakPtr()));
 }
@@ -96,10 +98,6 @@ void DownloadMediaParser::OnMediaMetadataParsed(
   metadata_ = std::move(metadata);
   DCHECK(metadata_);
 
-  // TODO(xingliu): Make |attached_images| movable and use this as a thumbnail
-  // source as well as video frame.
-  attached_images_ = attached_images;
-
   // For audio file, we only need metadata and poster.
   if (base::StartsWith(mime_type_, "audio/",
                        base::CompareCase::INSENSITIVE_ASCII)) {
@@ -128,22 +126,39 @@ void DownloadMediaParser::RetrieveEncodedVideoFrame() {
 
   media_parser()->ExtractVideoFrame(
       mime_type_, base::saturated_cast<uint32_t>(size_), std::move(source_ptr),
-      base::BindOnce(&DownloadMediaParser::OnEncodedVideoFrameRetrieved,
+      base::BindOnce(&DownloadMediaParser::OnVideoFrameRetrieved,
                      weak_factory_.GetWeakPtr()));
 }
 
-void DownloadMediaParser::OnEncodedVideoFrameRetrieved(
+void DownloadMediaParser::OnVideoFrameRetrieved(
     bool success,
-    const std::vector<uint8_t>& data,
+    chrome::mojom::VideoFrameDataPtr video_frame_data,
     const media::VideoDecoderConfig& config) {
-  if (data.empty()) {
+  if (!success) {
     OnError();
     return;
   }
 
-  encoded_data_ = data;
+  video_frame_data_ = std::move(video_frame_data);
   config_ = config;
 
+  // For vp8, vp9 codec, we directly do software decoding in utility process.
+  // Render now.
+  if (video_frame_data_->which() ==
+      chrome::mojom::VideoFrameData::Tag::DECODED_FRAME) {
+    decode_done_ = true;
+    RenderVideoFrame(std::move(video_frame_data_->get_decoded_frame()));
+    return;
+  }
+
+  // For other codec, the encoded frame is retrieved in utility process, send
+  // the data to GPU process to do hardware decoding.
+  if (video_frame_data_->get_encoded_data().empty()) {
+    OnError();
+    return;
+  }
+
+  // Starts to decode with MojoVideoDecoder.
   content::CreateGpuVideoAcceleratorFactories(base::BindRepeating(
       &DownloadMediaParser::OnGpuVideoAcceleratorFactoriesReady,
       weak_factory_.GetWeakPtr()));
@@ -162,82 +177,49 @@ void DownloadMediaParser::DecodeVideoFrame() {
 
   // Build and config the decoder.
   DCHECK(gpu_factories_);
-  decoder_ = std::make_unique<media::MojoVideoDecoder>(
+  auto mojo_decoder = std::make_unique<media::MojoVideoDecoder>(
       base::ThreadTaskRunnerHandle::Get(), gpu_factories_.get(), this,
       std::move(video_decoder_ptr), base::BindRepeating(&OnRequestOverlayInfo),
       gfx::ColorSpace());
 
-  decoder_->Initialize(
-      config_, false, nullptr,
-      base::BindRepeating(&DownloadMediaParser::OnVideoDecoderInitialized,
-                          weak_factory_.GetWeakPtr()),
-      base::BindRepeating(&DownloadMediaParser::OnVideoFrameDecoded,
-                          weak_factory_.GetWeakPtr()),
-      base::RepeatingClosure());
+  decoder_ = std::make_unique<media::VideoThumbnailDecoder>(
+      std::move(mojo_decoder), config_,
+      std::move(video_frame_data_->get_encoded_data()));
+
+  decoder_->Start(base::BindOnce(&DownloadMediaParser::OnVideoFrameDecoded,
+                                 weak_factory_.GetWeakPtr()));
 }
 
-void DownloadMediaParser::OnVideoDecoderInitialized(bool success) {
-  if (!success) {
+void DownloadMediaParser::OnVideoFrameDecoded(
+    scoped_refptr<media::VideoFrame> frame) {
+  if (!frame) {
     OnError();
     return;
   }
 
-  // Build the video buffer to decode.
-  auto buffer =
-      media::DecoderBuffer::CopyFrom(&encoded_data_[0], encoded_data_.size());
-  encoded_data_.clear();
-
-  // Decode one frame buffer, followed by eos buffer.
-  DCHECK_GE(decoder_->GetMaxDecodeRequests(), 2);
-  decoder_->Decode(
-      buffer, base::BindRepeating(&DownloadMediaParser::OnVideoBufferDecoded,
-                                  weak_factory_.GetWeakPtr()));
-  decoder_->Decode(media::DecoderBuffer::CreateEOSBuffer(),
-                   base::BindRepeating(&DownloadMediaParser::OnEosBufferDecoded,
-                                       weak_factory_.GetWeakPtr()));
-}
-
-void DownloadMediaParser::OnVideoBufferDecoded(media::DecodeStatus status) {
-  if (status != media::DecodeStatus::OK)
-    OnError();
-}
-
-void DownloadMediaParser::OnEosBufferDecoded(media::DecodeStatus status) {
-  if (status != media::DecodeStatus::OK)
-    OnError();
-
-  // Fails if no decoded video frame is generated when eos arrives.
-  if (!decode_done_)
-    OnError();
-}
-
-void DownloadMediaParser::OnVideoFrameDecoded(
-    const scoped_refptr<media::VideoFrame>& frame) {
-  DCHECK(frame);
   DCHECK(frame->HasTextures());
   decode_done_ = true;
 
-  RenderVideoFrame(frame);
+  RenderVideoFrame(std::move(frame));
 }
 
 void DownloadMediaParser::RenderVideoFrame(
-    const scoped_refptr<media::VideoFrame>& video_frame) {
-  auto converter = VideoFrameThumbnailConverter::Create(
-      config_.codec(), gpu_factories_->GetMediaContextProvider());
-  converter->ConvertToBitmap(
-      video_frame,
-      base::BindOnce(&DownloadMediaParser::OnBitmapGenerated,
-                     weak_factory_.GetWeakPtr(), std::move(converter)));
-}
-
-void DownloadMediaParser::OnBitmapGenerated(
-    std::unique_ptr<VideoFrameThumbnailConverter>,
-    bool success,
-    SkBitmap bitmap) {
-  if (!success) {
-    OnError();
-    return;
-  }
+    scoped_refptr<media::VideoFrame> video_frame) {
+  auto context_provider =
+      gpu_factories_ ? gpu_factories_->GetMediaContextProvider() : nullptr;
+
+  media::PaintCanvasVideoRenderer renderer;
+  SkBitmap bitmap;
+  bitmap.allocN32Pixels(video_frame->visible_rect().width(),
+                        video_frame->visible_rect().height());
+
+  // Draw the video frame to |bitmap|.
+  cc::SkiaPaintCanvas canvas(bitmap);
+  media::Context3D context =
+      context_provider ? media::Context3D(context_provider->ContextGL(),
+                                          context_provider->GrContext())
+                       : media::Context3D();
+  renderer.Copy(video_frame, &canvas, context);
 
   NotifyComplete(std::move(bitmap));
 }
@@ -275,8 +257,6 @@ void DownloadMediaParser::OnMediaDataReady(
 }
 
 void DownloadMediaParser::NotifyComplete(SkBitmap bitmap) {
-  // TODO(xingliu): Return the metadata and video thumbnail data in
-  // |parse_complete_cb_|.
   DCHECK(metadata_);
   if (parse_complete_cb_)
     std::move(parse_complete_cb_)

chrome/browser/android/download/download_media_parser.h

@@ -26,10 +26,10 @@ class GpuVideoAcceleratorFactories;
 class MediaInterfaceProvider;
 class MojoVideoDecoder;
 class VideoDecoderConfig;
+class VideoThumbnailDecoder;
 }  // namespace media
 
 class SkBitmap;
-class VideoFrameThumbnailConverter;
 
 // Parse local media files, including media metadata and thumbnails.
 // Metadata is always parsed in utility process for both audio and video files.
@@ -69,27 +69,21 @@ class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
 
   // Retrieves an encoded video frame.
   void RetrieveEncodedVideoFrame();
-  void OnEncodedVideoFrameRetrieved(bool success,
-                                    const std::vector<uint8_t>& data,
-                                    const media::VideoDecoderConfig& config);
+  void OnVideoFrameRetrieved(bool success,
+                             chrome::mojom::VideoFrameDataPtr video_frame_data,
+                             const media::VideoDecoderConfig& config);
 
   // Decodes the video frame.
   void OnGpuVideoAcceleratorFactoriesReady(
       std::unique_ptr<media::GpuVideoAcceleratorFactories>);
   void DecodeVideoFrame();
-  void OnVideoDecoderInitialized(bool success);
-  void OnVideoBufferDecoded(media::DecodeStatus status);
-  void OnEosBufferDecoded(media::DecodeStatus status);
-  void OnVideoFrameDecoded(
-      const scoped_refptr<media::VideoFrame>& decoded_frame);
-  media::mojom::InterfaceFactory* GetMediaInterfaceFactory();
-  void OnDecoderConnectionError();
+  void OnVideoFrameDecoded(scoped_refptr<media::VideoFrame> decoded_frame);
 
   // Renders the video frame to bitmap.
-  void RenderVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame);
-  void OnBitmapGenerated(std::unique_ptr<VideoFrameThumbnailConverter>,
-                         bool success,
-                         SkBitmap bitmap);
+  void RenderVideoFrame(scoped_refptr<media::VideoFrame> video_frame);
+
+  media::mojom::InterfaceFactory* GetMediaInterfaceFactory();
+  void OnDecoderConnectionError();
 
   // Overlays media data source read operation. Gradually read data from media
   // file.
@@ -106,21 +100,21 @@ class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
   ParseCompleteCB parse_complete_cb_;
   chrome::mojom::MediaMetadataPtr metadata_;
 
-  // Poster images obtained with |metadata_|.
-  std::vector<metadata::AttachedImage> attached_images_;
-
+  // Used to read media files chunks to feed to IPC channel.
   std::unique_ptr<chrome::mojom::MediaDataSource> media_data_source_;
 
   // The task runner to do blocking disk IO.
   scoped_refptr<base::SequencedTaskRunner> file_task_runner_;
 
-  // Encoded video frame to be decoded. This data can be large for high
-  // resolution video, should be std::move or cleared whenever possible.
-  std::vector<uint8_t> encoded_data_;
+  // Cached video frame data, which contains either encoded frame or decoded
+  // video frame. Encoded frame is extracted with ffmpeg, the data can be large
+  // for high resolution video.
+  chrome::mojom::VideoFrameDataPtr video_frame_data_;
 
-  // Objects used to decode the video into media::VideoFrame.
+  // Objects used to decode the video into media::VideoFrame with
+  // MojoVideoDecoder.
   media::VideoDecoderConfig config_;
-  std::unique_ptr<media::MojoVideoDecoder> decoder_;
+  std::unique_ptr<media::VideoThumbnailDecoder> decoder_;
   media::mojom::InterfaceFactoryPtr media_interface_factory_;
   std::unique_ptr<media::MediaInterfaceProvider> media_interface_provider_;
   std::unique_ptr<media::GpuVideoAcceleratorFactories> gpu_factories_;

chrome/browser/android/download/video_frame_thumbnail_converter.cc

-// Copyright 2018 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "chrome/browser/android/download/video_frame_thumbnail_converter.h"
-
-#include "base/macros.h"
-#include "cc/paint/skia_paint_canvas.h"
-#include "components/viz/common/gl_helper.h"
-#include "media/base/video_frame.h"
-#include "media/renderers/paint_canvas_video_renderer.h"
-#include "services/ws/public/cpp/gpu/context_provider_command_buffer.h"
-
-namespace {
-
-// Class to generate bitmap based on video frame backed by texture in remote
-// process.
-class TextureFrameThumbnailConverter : public VideoFrameThumbnailConverter {
- public:
-  TextureFrameThumbnailConverter(
-      scoped_refptr<ws::ContextProviderCommandBuffer> context_provider)
-      : context_provider_(std::move(context_provider)) {}
-  TextureFrameThumbnailConverter() = default;
-  ~TextureFrameThumbnailConverter() override = default;
-
- private:
-  // VideoFrameThumbnailConverter implementation.
-  void ConvertToBitmap(const scoped_refptr<media::VideoFrame>& frame,
-                       BitmapCallback pixel_callback) override {
-    DCHECK(frame->HasTextures())
-        << "This implementation only do texture read backs.";
-
-    // Read back the texture data contained in the video frame.
-    media::PaintCanvasVideoRenderer renderer;
-    SkBitmap skbitmap;
-    skbitmap.allocN32Pixels(frame->visible_rect().width(),
-                            frame->visible_rect().height());
-    cc::SkiaPaintCanvas canvas(skbitmap);
-    renderer.Copy(frame, &canvas,
-                  media::Context3D(context_provider_->ContextGL(),
-                                   context_provider_->GrContext()));
-
-    std::move(pixel_callback).Run(true, std::move(skbitmap));
-  }
-
-  // Command buffer channel used to read back the pixel data from texture in
-  // remote process.
-  scoped_refptr<ws::ContextProviderCommandBuffer> context_provider_;
-
-  DISALLOW_COPY_AND_ASSIGN(TextureFrameThumbnailConverter);
-};
-
-}  // namespace
-
-// static
-std::unique_ptr<VideoFrameThumbnailConverter>
-VideoFrameThumbnailConverter::Create(
-    media::VideoCodec codec,
-    scoped_refptr<ws::ContextProviderCommandBuffer> context_provider) {
-  // TODO(xingliu): Implement vpx decode and render pipeline.
-  DCHECK_NE(codec, media::VideoCodec::kCodecVP8);
-  DCHECK_NE(codec, media::VideoCodec::kCodecVP9);
-
-  return std::make_unique<TextureFrameThumbnailConverter>(
-      std::move(context_provider));
-}

chrome/browser/android/download/video_frame_thumbnail_converter.h

-// Copyright 2018 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef CHROME_BROWSER_ANDROID_DOWNLOAD_VIDEO_FRAME_THUMBNAIL_CONVERTER_H_
-#define CHROME_BROWSER_ANDROID_DOWNLOAD_VIDEO_FRAME_THUMBNAIL_CONVERTER_H_
-
-#include <memory>
-#include <vector>
-
-#include "base/callback.h"
-#include "base/memory/scoped_refptr.h"
-#include "media/base/video_codecs.h"
-
-namespace media {
-class VideoFrame;
-}  // namespace media
-
-namespace ws {
-class ContextProviderCommandBuffer;
-}  // namespace ws
-
-class SkBitmap;
-
-// Generates video thumbnail from a video frame. For most video format such as
-// H264, the input video frame contains a texture created in GPU process, we do
-// a readback and generate the bitmap. For vp8, vp9 format, the video frame
-// contains in-memory YUV data.
-class VideoFrameThumbnailConverter {
- public:
-  using BitmapCallback = base::OnceCallback<void(bool, SkBitmap)>;
-
-  // Create the video thumbnail renderer for video type with |codec|. The
-  // decoder to generate video frame may yields video frames backed by texture
-  // in remote process or raw YUV data.
-  static std::unique_ptr<VideoFrameThumbnailConverter> Create(
-      media::VideoCodec codec,
-      scoped_refptr<ws::ContextProviderCommandBuffer> context_provider);
-
-  virtual void ConvertToBitmap(const scoped_refptr<media::VideoFrame>& frame,
-                               BitmapCallback pixel_callback) = 0;
-
-  virtual ~VideoFrameThumbnailConverter() = default;
-};
-
-#endif  // CHROME_BROWSER_ANDROID_DOWNLOAD_VIDEO_FRAME_THUMBNAIL_CONVERTER_H_

chrome/services/media_gallery_util/BUILD.gn

@@ -55,6 +55,8 @@ if (is_android) {
 
     data = [
       "//media/test/data/bear.mp4",
+      "//media/test/data/bear-vp8-webvtt.webm",
+      "//media/test/data/bear-vp8a.webm",
       "//media/test/data/sfx.mp3",
     ]
 

chrome/services/media_gallery_util/media_parser_android.cc

@@ -8,25 +8,85 @@
 #include "base/task/task_traits.h"
 #include "chrome/services/media_gallery_util/ipc_data_source.h"
 #include "media/base/bind_to_current_loop.h"
+#include "media/base/video_codecs.h"
+#include "media/base/video_thumbnail_decoder.h"
 #include "media/filters/android/video_frame_extractor.h"
+#include "media/filters/vpx_video_decoder.h"
+#include "media/media_buildflags.h"
+#include "media/mojo/common/mojo_shared_buffer_video_frame.h"
 
 namespace {
 
-void OnThumbnailGenerated(
+// Return the video frame back to browser process. A valid |config| is
+// needed for deserialization.
+void OnSoftwareVideoFrameDecoded(
+    std::unique_ptr<media::VideoThumbnailDecoder>,
+    MediaParser::ExtractVideoFrameCallback video_frame_callback,
+    const media::VideoDecoderConfig& config,
+    scoped_refptr<media::VideoFrame> frame) {
+  DCHECK(video_frame_callback);
+
+  if (!frame) {
+    std::move(video_frame_callback)
+        .Run(false, chrome::mojom::VideoFrameData::New(), config);
+    return;
+  }
+
+  std::move(video_frame_callback)
+      .Run(true,
+           chrome::mojom::VideoFrameData::NewDecodedFrame(
+               media::MojoSharedBufferVideoFrame::CreateFromYUVFrame(*frame)),
+           config);
+}
+
+void OnEncodedVideoFrameExtracted(
     std::unique_ptr<media::VideoFrameExtractor> video_frame_extractor,
-    MediaParser::ExtractVideoFrameCallback extract_frame_cb,
+    MediaParser::ExtractVideoFrameCallback video_frame_callback,
     bool success,
-    const std::vector<uint8_t>& data,
+    std::vector<uint8_t> data,
     const media::VideoDecoderConfig& config) {
-  std::move(extract_frame_cb).Run(success, data, config);
+  if (!success || data.empty()) {
+    std::move(video_frame_callback)
+        .Run(false, chrome::mojom::VideoFrameData::New(), config);
+    return;
+  }
+
+#if BUILDFLAG(USE_PROPRIETARY_CODECS) && \
+    !BUILDFLAG(ENABLE_FFMPEG_VIDEO_DECODERS)
+  // H264 currently needs to be decoded in GPU process when no software decoder
+  // is provided.
+  if (config.codec() == media::VideoCodec::kCodecH264) {
+    std::move(video_frame_callback)
+        .Run(success,
+             chrome::mojom::VideoFrameData::NewEncodedData(std::move(data)),
+             config);
+    return;
+  }
+#endif
+
+  if (config.codec() != media::VideoCodec::kCodecVP8 &&
+      config.codec() != media::VideoCodec::kCodecVP9) {
+    std::move(video_frame_callback)
+        .Run(false, chrome::mojom::VideoFrameData::New(), config);
+    return;
+  }
+
+  // Decode with libvpx for vp8, vp9.
+  auto thumbnail_decoder = std::make_unique<media::VideoThumbnailDecoder>(
+      std::make_unique<media::VpxVideoDecoder>(), config, std::move(data));
+
+  thumbnail_decoder->Start(
+      base::BindOnce(&OnSoftwareVideoFrameDecoded, std::move(thumbnail_decoder),
+                     std::move(video_frame_callback), config));
 }
 
 void ExtractVideoFrameOnMediaThread(
     media::DataSource* data_source,
-    MediaParser::ExtractVideoFrameCallback extract_frame_callback) {
+    MediaParser::ExtractVideoFrameCallback video_frame_callback) {
   auto extractor = std::make_unique<media::VideoFrameExtractor>(data_source);
-  extractor->Start(base::BindOnce(&OnThumbnailGenerated, std::move(extractor),
-                                  std::move(extract_frame_callback)));
+  extractor->Start(base::BindOnce(&OnEncodedVideoFrameExtracted,
+                                  std::move(extractor),
+                                  std::move(video_frame_callback)));
 }
 
 }  // namespace
@@ -43,7 +103,7 @@ void MediaParserAndroid::ExtractVideoFrame(
     const std::string& mime_type,
     uint32_t total_size,
     chrome::mojom::MediaDataSourcePtr media_data_source,
-    MediaParser::ExtractVideoFrameCallback extract_frame_callback) {
+    MediaParser::ExtractVideoFrameCallback video_frame_callback) {
   data_source_ = std::make_unique<IPCDataSource>(
       std::move(media_data_source), static_cast<int64_t>(total_size));
 
@@ -51,5 +111,5 @@ void MediaParserAndroid::ExtractVideoFrame(
       FROM_HERE,
       base::BindOnce(
           &ExtractVideoFrameOnMediaThread, data_source_.get(),
-          media::BindToCurrentLoop(std::move(extract_frame_callback))));
+          media::BindToCurrentLoop(std::move(video_frame_callback))));
 }

chrome/services/media_gallery_util/media_parser_android.h

@@ -22,10 +22,11 @@ class MediaParserAndroid : public MediaParser {
   ~MediaParserAndroid() override;
 
   // MediaParser implementation.
-  void ExtractVideoFrame(const std::string& mime_type,
-                         uint32_t total_size,
-                         chrome::mojom::MediaDataSourcePtr media_data_source,
-                         ExtractVideoFrameCallback callback) override;
+  void ExtractVideoFrame(
+      const std::string& mime_type,
+      uint32_t total_size,
+      chrome::mojom::MediaDataSourcePtr media_data_source,
+      ExtractVideoFrameCallback video_frame_callback) override;
 
  private:
   // The task runner to do blocking IO. The utility thread cannot be blocked.

chrome/services/media_gallery_util/media_parser_android_unittest.cc

@@ -15,12 +15,41 @@
 #include "base/test/scoped_task_environment.h"
 #include "chrome/services/media_gallery_util/public/mojom/media_parser.mojom.h"
 #include "media/base/test_data_util.h"
+#include "media/media_buildflags.h"
 #include "mojo/public/cpp/bindings/binding.h"
 #include "services/service_manager/public/cpp/service_context_ref.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace {
 
+struct ExtractVideoFrameResult {
+  bool success = false;
+  chrome::mojom::VideoFrameDataPtr video_frame_data;
+  media::VideoDecoderConfig config;
+};
+
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+// Returns if the first 3 or 4 bytes of H264 encoded |data| is the start code,
+// 0x000001 or 0x00000001.
+bool HasH264StartCode(const std::vector<uint8_t>& data) {
+  if (data.size() < 4 || (data[0] != 0u || data[1] != 0u))
+    return false;
+  return data[2] == 0x01 || (data[2] == 0u && data[3] == 0x01);
+}
+#endif
+
+// Returns if the first few bytes in the YUV frame are not all zero. This is a
+// rough method to verify the frame is not empty.
+bool HasValidYUVData(const scoped_refptr<media::VideoFrame>& frame) {
+  bool valid = false;
+  for (size_t i = 0; i < 8; ++i) {
+    valid |= *(frame->data(media::VideoFrame::kYPlane) + i);
+    if (valid)
+      break;
+  }
+  return valid;
+}
+
 // Used in test that do blocking reads from a local file.
 class TestMediaDataSource : public chrome::mojom::MediaDataSource {
  public:
@@ -65,6 +94,33 @@ class MediaParserAndroidTest : public testing::Test {
  protected:
   MediaParserAndroid* parser() { return parser_.get(); }
 
+  ExtractVideoFrameResult ExtractFrame(const std::string& file_name,
+                                       const std::string& mime_type) {
+    auto file_path = media::GetTestDataFilePath(file_name);
+    int64_t size = 0;
+    EXPECT_TRUE(base::GetFileSize(file_path, &size));
+
+    chrome::mojom::MediaDataSourcePtr data_source_ptr;
+    TestMediaDataSource test_data_source(&data_source_ptr, file_path);
+
+    ExtractVideoFrameResult result;
+    base::RunLoop run_loop;
+    parser()->ExtractVideoFrame(
+        mime_type, size, std::move(data_source_ptr),
+        base::BindLambdaForTesting(
+            [&](bool success, chrome::mojom::VideoFrameDataPtr video_frame_data,
+                const media::VideoDecoderConfig& config) {
+              result.success = success;
+              result.video_frame_data = std::move(video_frame_data);
+              result.config = config;
+
+              run_loop.Quit();
+            }));
+    run_loop.Run();
+
+    return result;
+  }
+
  private:
   std::unique_ptr<MediaParserAndroid> parser_;
 
@@ -74,26 +130,50 @@ class MediaParserAndroidTest : public testing::Test {
   DISALLOW_COPY_AND_ASSIGN(MediaParserAndroidTest);
 };
 
-// Test to verify encoded video frame can be extracted.
-TEST_F(MediaParserAndroidTest, VideoFrameExtraction) {
-  auto file_path = media::GetTestDataFilePath("bear.mp4");
-  int64_t size = 0;
-  ASSERT_TRUE(base::GetFileSize(file_path, &size));
-
-  chrome::mojom::MediaDataSourcePtr data_source_ptr;
-  TestMediaDataSource test_data_source(&data_source_ptr, file_path);
-
-  bool result = false;
-  base::RunLoop run_loop;
-  parser()->ExtractVideoFrame(
-      "video/mp4", size, std::move(data_source_ptr),
-      base::BindLambdaForTesting([&](bool success, const std::vector<uint8_t>&,
-                                     const media::VideoDecoderConfig&) {
-        result = success;
-        run_loop.Quit();
-      }));
-  run_loop.Run();
-  EXPECT_TRUE(result);
+#if BUILDFLAG(USE_PROPRIETARY_CODECS)
+// Test to verify an encoded video frame can be extracted for h264 codec video
+// file. Decoding needs to happen in other process.
+TEST_F(MediaParserAndroidTest, VideoFrameExtractionH264) {
+  auto result = ExtractFrame("bear.mp4", "video/mp4");
+  EXPECT_TRUE(result.success);
+  EXPECT_EQ(result.video_frame_data->which(),
+            chrome::mojom::VideoFrameData::Tag::ENCODED_DATA);
+  EXPECT_FALSE(result.video_frame_data->get_encoded_data().empty());
+  EXPECT_TRUE(HasH264StartCode(result.video_frame_data->get_encoded_data()));
+}
+#endif
+
+// Test to verify a decoded video frame can be extracted for vp8 codec video
+// file with YUV420 color format.
+TEST_F(MediaParserAndroidTest, VideoFrameExtractionVp8) {
+  auto result = ExtractFrame("bear-vp8-webvtt.webm", "video/webm");
+  EXPECT_TRUE(result.success);
+  EXPECT_EQ(result.video_frame_data->which(),
+            chrome::mojom::VideoFrameData::Tag::DECODED_FRAME);
+  const auto& frame = result.video_frame_data->get_decoded_frame();
+  EXPECT_TRUE(frame);
+  EXPECT_TRUE(HasValidYUVData(frame));
+  EXPECT_TRUE(frame->IsMappable());
+  EXPECT_FALSE(frame->HasTextures());
+  EXPECT_EQ(frame->storage_type(),
+            media::VideoFrame::StorageType::STORAGE_MOJO_SHARED_BUFFER);
+}
+
+// Test to verify a decoded video frame can be extracted for vp8 codec with
+// alpha plane.
+TEST_F(MediaParserAndroidTest, VideoFrameExtractionVp8WithAlphaPlane) {
+  auto result = ExtractFrame("bear-vp8a.webm", "video/webm");
+  EXPECT_TRUE(result.success);
+
+  EXPECT_EQ(result.video_frame_data->which(),
+            chrome::mojom::VideoFrameData::Tag::DECODED_FRAME);
+  const auto& frame = result.video_frame_data->get_decoded_frame();
+  EXPECT_TRUE(frame);
+  EXPECT_TRUE(HasValidYUVData(frame));
+  EXPECT_TRUE(frame->IsMappable());
+  EXPECT_FALSE(frame->HasTextures());
+  EXPECT_EQ(frame->storage_type(),
+            media::VideoFrame::StorageType::STORAGE_MOJO_SHARED_BUFFER);
 }
 
 }  // namespace

chrome/services/media_gallery_util/public/mojom/media_parser.mojom

@@ -9,6 +9,13 @@ import "mojo/public/mojom/base/file.mojom";
 import "mojo/public/mojom/base/time.mojom";
 import "mojo/public/mojom/base/values.mojom";
 
+// Contains either encoded video frame in |encoded_data|, or decoded video
+// frame in |decoded_frame|.
+union VideoFrameData {
+  array<uint8> encoded_data;
+  media.mojom.VideoFrame? decoded_frame;
+};
+
 interface MediaParser {
   // Extracts metadata from media data with |mime_type|, |total_size| and
   // available from |media_data_source|. If there are images referred to in the
@@ -22,14 +29,15 @@ interface MediaParser {
           MediaMetadata metadata,
           array<AttachedImage> attached_images);
 
-  // Extracts one video key frame from |media_data_source|. The frame is still
-  // in encoded format.
+  // Extracts one video key frame from |media_data_source|. Returns the decoded
+  // frame if decoding can be done in utility process, or an encoded frame if
+  // decoding needs to happen in other process.
   [EnableIf=is_android]
   ExtractVideoFrame(string mime_type,
                     uint32 total_size,
                     MediaDataSource media_data_source)
       => (bool success,
-          array<uint8> data,
+          VideoFrameData frame_data,
           media.mojom.VideoDecoderConfig config);
 
   // Validates the passed media file with sanity checks, and file decoding

media/base/BUILD.gn

@@ -278,6 +278,8 @@ jumbo_source_set("base") {
     "video_renderer.h",
     "video_rotation.cc",
     "video_rotation.h",
+    "video_thumbnail_decoder.cc",
+    "video_thumbnail_decoder.h",
     "video_types.cc",
     "video_types.h",
     "video_util.cc",
@@ -521,6 +523,7 @@ source_set("unit_tests") {
     "video_frame_layout_unittest.cc",
     "video_frame_pool_unittest.cc",
     "video_frame_unittest.cc",
+    "video_thumbnail_decoder_unittest.cc",
     "video_types_unittest.cc",
     "video_util_unittest.cc",
     "wall_clock_time_source_unittest.cc",

media/base/video_thumbnail_decoder.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/base/video_thumbnail_decoder.h"
+
+#include "base/bind_helpers.h"
+#include "media/base/decoder_buffer.h"
+#include "media/base/video_frame.h"
+
+namespace media {
+
+VideoThumbnailDecoder::VideoThumbnailDecoder(
+    std::unique_ptr<VideoDecoder> decoder,
+    const VideoDecoderConfig& config,
+    std::vector<uint8_t> encoded_data)
+    : decoder_(std::move(decoder)),
+      config_(config),
+      encoded_data_(std::move(encoded_data)),
+      weak_factory_(this) {
+  DCHECK(!encoded_data_.empty());
+  DCHECK(config_.IsValidConfig());
+}
+
+VideoThumbnailDecoder::~VideoThumbnailDecoder() = default;
+
+void VideoThumbnailDecoder::Start(VideoFrameCallback video_frame_callback) {
+  video_frame_callback_ = std::move(video_frame_callback);
+  DCHECK(video_frame_callback_);
+  decoder_->Initialize(
+      config_, false, nullptr,
+      base::BindRepeating(&VideoThumbnailDecoder::OnVideoDecoderInitialized,
+                          weak_factory_.GetWeakPtr()),
+      base::BindRepeating(&VideoThumbnailDecoder::OnVideoFrameDecoded,
+                          weak_factory_.GetWeakPtr()),
+      base::DoNothing());
+}
+
+void VideoThumbnailDecoder::OnVideoDecoderInitialized(bool success) {
+  if (!success) {
+    NotifyComplete(nullptr);
+    return;
+  }
+
+  auto buffer =
+      DecoderBuffer::CopyFrom(&encoded_data_[0], encoded_data_.size());
+  encoded_data_.clear();
+  decoder_->Decode(
+      buffer, base::BindRepeating(&VideoThumbnailDecoder::OnVideoBufferDecoded,
+                                  weak_factory_.GetWeakPtr()));
+}
+
+void VideoThumbnailDecoder::OnVideoBufferDecoded(DecodeStatus status) {
+  if (status != DecodeStatus::OK) {
+    NotifyComplete(nullptr);
+    return;
+  }
+
+  // Enqueue eos since only one video frame is needed for thumbnail.
+  decoder_->Decode(
+      DecoderBuffer::CreateEOSBuffer(),
+      base::BindRepeating(&VideoThumbnailDecoder::OnEosBufferDecoded,
+                          weak_factory_.GetWeakPtr()));
+}
+
+void VideoThumbnailDecoder::OnEosBufferDecoded(DecodeStatus status) {
+  if (status != DecodeStatus::OK)
+    NotifyComplete(nullptr);
+}
+
+void VideoThumbnailDecoder::OnVideoFrameDecoded(
+    const scoped_refptr<VideoFrame>& frame) {
+  NotifyComplete(std::move(frame));
+}
+
+void VideoThumbnailDecoder::NotifyComplete(scoped_refptr<VideoFrame> frame) {
+  DCHECK(video_frame_callback_);
+  std::move(video_frame_callback_).Run(std::move(frame));
+}
+
+}  // namespace media

media/base/video_thumbnail_decoder.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_BASE_VIDEO_THUMBNAIL_DECODER_H_
+#define MEDIA_BASE_VIDEO_THUMBNAIL_DECODER_H_
+
+#include <memory>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/macros.h"
+#include "base/memory/ref_counted.h"
+#include "base/memory/weak_ptr.h"
+#include "media/base/media_export.h"
+#include "media/base/video_decoder.h"
+#include "media/base/video_decoder_config.h"
+
+namespace media {
+
+class VideoFrame;
+
+// Class to decode a single video frame for video thumbnail generation.
+class MEDIA_EXPORT VideoThumbnailDecoder {
+ public:
+  using VideoFrameCallback =
+      base::OnceCallback<void(scoped_refptr<VideoFrame>)>;
+
+  // Creates the thumbnail decoder with |decoder| that performs the actual work.
+  // |encoded_data| is the encoded frame extracted with ffmpeg. A video frame
+  // will be returned through |video_frame_callback|.
+  VideoThumbnailDecoder(std::unique_ptr<VideoDecoder> decoder,
+                        const VideoDecoderConfig& config,
+                        std::vector<uint8_t> encoded_data);
+  ~VideoThumbnailDecoder();
+
+  // Starts to decode the video frame.
+  void Start(VideoFrameCallback video_frame_callback);
+
+ private:
+  void OnVideoDecoderInitialized(bool success);
+  void OnVideoBufferDecoded(DecodeStatus status);
+  void OnEosBufferDecoded(DecodeStatus status);
+
+  // Called when the output frame is generated.
+  void OnVideoFrameDecoded(const scoped_refptr<VideoFrame>& frame);
+
+  void NotifyComplete(scoped_refptr<VideoFrame> frame);
+
+  // The decoder that does the actual decoding.
+  std::unique_ptr<VideoDecoder> decoder_;
+  const VideoDecoderConfig config_;
+
+  // The demuxed, encoded video frame waiting to be decoded.
+  std::vector<uint8_t> encoded_data_;
+
+  VideoFrameCallback video_frame_callback_;
+  base::WeakPtrFactory<VideoThumbnailDecoder> weak_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(VideoThumbnailDecoder);
+};
+
+}  // namespace media
+
+#endif  // MEDIA_BASE_VIDEO_THUMBNAIL_DECODER_H_

media/base/video_thumbnail_decoder_unittest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <memory>
+#include <vector>
+
+#include "base/bind_helpers.h"
+#include "base/run_loop.h"
+#include "base/test/scoped_task_environment.h"
+#include "media/base/gmock_callback_support.h"
+#include "media/base/media_util.h"
+#include "media/base/mock_filters.h"
+#include "media/base/video_decoder_config.h"
+#include "media/base/video_frame.h"
+#include "media/base/video_thumbnail_decoder.h"
+#include "media/base/video_types.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+using ::testing::_;
+using ::testing::DoAll;
+
+namespace media {
+namespace {
+
+class VideoThumbnailDecoderTest : public testing::Test {
+ public:
+  VideoThumbnailDecoderTest() {}
+  ~VideoThumbnailDecoderTest() override {}
+
+ protected:
+  void SetUp() override {
+    auto mock_video_decoder = std::make_unique<MockVideoDecoder>();
+    mock_video_decoder_ = mock_video_decoder.get();
+    VideoDecoderConfig valid_config(
+        kCodecVP8, VP8PROFILE_ANY, PIXEL_FORMAT_I420, COLOR_SPACE_UNSPECIFIED,
+        VIDEO_ROTATION_0, gfx::Size(1, 1), gfx::Rect(1, 1), gfx::Size(1, 1),
+        EmptyExtraData(), Unencrypted());
+
+    thumbnail_decoder_ = std::make_unique<VideoThumbnailDecoder>(
+        std::move(mock_video_decoder), valid_config, std::vector<uint8_t>{0u});
+  }
+
+  void TearDown() override {}
+
+  void Start() {
+    thumbnail_decoder_->Start(base::BindOnce(
+        &VideoThumbnailDecoderTest::OnFrameDecoded, base::Unretained(this)));
+    base::RunLoop().RunUntilIdle();
+  }
+
+  scoped_refptr<VideoFrame> CreateFrame() {
+    return VideoFrame::CreateZeroInitializedFrame(
+        VideoPixelFormat::PIXEL_FORMAT_I420, gfx::Size(1, 1), gfx::Rect(1, 1),
+        gfx::Size(1, 1), base::TimeDelta());
+  }
+
+  VideoThumbnailDecoder* thumbnail_decoder() {
+    return thumbnail_decoder_.get();
+  }
+  MockVideoDecoder* mock_video_decoder() { return mock_video_decoder_; }
+  const scoped_refptr<VideoFrame>& frame() { return frame_; }
+
+ private:
+  void OnFrameDecoded(scoped_refptr<VideoFrame> frame) {
+    frame_ = std::move(frame);
+  }
+
+  base::test::ScopedTaskEnvironment scoped_task_environment_;
+
+  MockVideoDecoder* mock_video_decoder_;
+  std::unique_ptr<VideoThumbnailDecoder> thumbnail_decoder_;
+
+  // The video frame returned from the thumbnail decoder.
+  scoped_refptr<VideoFrame> frame_;
+
+  DISALLOW_COPY_AND_ASSIGN(VideoThumbnailDecoderTest);
+};
+
+// Verifies a video frame can be delivered when decoder successfully created
+// the video frame.
+TEST_F(VideoThumbnailDecoderTest, Success) {
+  auto expected_frame = CreateFrame();
+  EXPECT_CALL(*mock_video_decoder(), Initialize(_, _, _, _, _, _))
+      .WillOnce(DoAll(RunCallback<3>(true), RunCallback<4>(expected_frame)));
+  EXPECT_CALL(*mock_video_decoder(), Decode(_, _))
+      .Times(2)
+      .WillRepeatedly(RunCallback<1>(DecodeStatus::OK));
+
+  Start();
+  EXPECT_TRUE(frame());
+}
+
+// No output video frame when decoder failed to initialize.
+TEST_F(VideoThumbnailDecoderTest, InitializationFailed) {
+  auto expected_frame = CreateFrame();
+  EXPECT_CALL(*mock_video_decoder(), Initialize(_, _, _, _, _, _))
+      .WillOnce(RunCallback<3>(false));
+
+  Start();
+  EXPECT_FALSE(frame());
+}
+
+// No output video frame when decoder failed to decode.
+TEST_F(VideoThumbnailDecoderTest, DecodingFailed) {
+  auto expected_frame = CreateFrame();
+  EXPECT_CALL(*mock_video_decoder(), Initialize(_, _, _, _, _, _))
+      .WillOnce(RunCallback<3>(true));
+  EXPECT_CALL(*mock_video_decoder(), Decode(_, _))
+      .WillOnce(RunCallback<1>(DecodeStatus::DECODE_ERROR));
+
+  Start();
+  EXPECT_FALSE(frame());
+}
+
+}  // namespace
+}  // namespace media

media/filters/android/video_frame_extractor.cc

@@ -107,7 +107,8 @@ void VideoFrameExtractor::ConvertPacket(AVPacket* packet) {
       break;
   }
 
-  bitstream_converter_->ConvertPacket(packet);
+  if (bitstream_converter_)
+    bitstream_converter_->ConvertPacket(packet);
 #endif  // BUILDFLAG(USE_PROPRIETARY_CODECS)
 }
 

media/filters/android/video_frame_extractor.h

@@ -38,7 +38,7 @@ class MEDIA_EXPORT VideoFrameExtractor {
  public:
   using VideoFrameCallback =
       base::OnceCallback<void(bool success,
-                              const std::vector<uint8_t>& data,
+                              std::vector<uint8_t> data,
                               const VideoDecoderConfig& decoder_config)>;
 
   explicit VideoFrameExtractor(DataSource* data_source);

media/filters/android/video_frame_extractor_unittest.cc

@@ -23,13 +23,13 @@ struct ExtractVideoFrameResult {
   VideoDecoderConfig decoder_config;
 };
 
-void OnThumbnailGenerated(ExtractVideoFrameResult* result,
-                          base::RepeatingClosure quit_closure,
-                          bool success,
-                          const std::vector<uint8_t>& encoded_frame,
-                          const VideoDecoderConfig& decoder_config) {
+void OnFrameExtracted(ExtractVideoFrameResult* result,
+                      base::RepeatingClosure quit_closure,
+                      bool success,
+                      std::vector<uint8_t> encoded_frame,
+                      const VideoDecoderConfig& decoder_config) {
   result->success = success;
-  result->encoded_frame = encoded_frame;
+  result->encoded_frame = std::move(encoded_frame);
   result->decoder_config = decoder_config;
 
   quit_closure.Run();
@@ -65,7 +65,7 @@ TEST_F(VideoFrameExtractorTest, ExtractVideoFrame) {
   ExtractVideoFrameResult result;
   base::RunLoop loop;
   extractor()->Start(
-      base::BindOnce(&OnThumbnailGenerated, &result, loop.QuitClosure()));
+      base::BindOnce(&OnFrameExtracted, &result, loop.QuitClosure()));
   loop.Run();
   EXPECT_TRUE(result.success);
   EXPECT_GT(result.encoded_frame.size(), 0u);

media/mojo/common/mojo_shared_buffer_video_frame.cc

@@ -13,6 +13,7 @@
 #include "base/logging.h"
 #include "base/numerics/safe_conversions.h"
 #include "base/numerics/safe_math.h"
+#include "mojo/public/cpp/system/platform_handle.h"
 
 namespace media {
 
@@ -56,6 +57,64 @@ MojoSharedBufferVideoFrame::CreateDefaultI420(const gfx::Size& dimensions,
                 coded_size.width() / 2, coded_size.width() / 2, timestamp);
 }
 
+scoped_refptr<MojoSharedBufferVideoFrame>
+MojoSharedBufferVideoFrame::CreateFromYUVFrame(const VideoFrame& frame) {
+  DCHECK_EQ(VideoFrame::NumPlanes(frame.format()), 3u);
+
+  // The data from |frame| may not be consecutive between planes. Copy data
+  // into a shared memory buffer which is tightly packed.
+  size_t allocation_size =
+      VideoFrame::AllocationSize(frame.format(), frame.coded_size());
+  mojo::ScopedSharedBufferHandle handle =
+      mojo::SharedBufferHandle::Create(allocation_size);
+
+  const size_t y_size =
+      VideoFrame::PlaneSize(frame.format(), VideoFrame::kYPlane,
+                            frame.coded_size())
+          .GetArea();
+  const size_t u_size =
+      VideoFrame::PlaneSize(frame.format(), VideoFrame::kUPlane,
+                            frame.coded_size())
+          .GetArea();
+  const size_t v_size =
+      VideoFrame::PlaneSize(frame.format(), VideoFrame::kVPlane,
+                            frame.coded_size())
+          .GetArea();
+
+  // Computes the offset of planes in shared memory buffer.
+  const size_t y_offset = 0u;
+  const size_t u_offset = y_offset + y_size;
+  const size_t v_offset = y_offset + y_size + u_size;
+
+  // Create a mojo video frame backed by shared memory, so it can be sent to
+  // the browser process.
+  scoped_refptr<MojoSharedBufferVideoFrame> mojo_frame =
+      MojoSharedBufferVideoFrame::Create(
+          frame.format(), frame.coded_size(), frame.visible_rect(),
+          frame.natural_size(), std::move(handle), allocation_size, y_offset,
+          u_offset, v_offset, frame.stride(VideoFrame::kYPlane),
+          frame.stride(VideoFrame::kUPlane), frame.stride(VideoFrame::kVPlane),
+          frame.timestamp());
+
+  // Copy Y plane.
+  memcpy(mojo_frame->shared_buffer_data(),
+         static_cast<const void*>(frame.data(VideoFrame::kYPlane)), y_size);
+
+  // Copy U plane.
+  memcpy(mojo_frame->shared_buffer_data() + u_offset,
+         static_cast<const void*>(frame.data(VideoFrame::kUPlane)), u_size);
+
+  // Copy V plane.
+  memcpy(mojo_frame->shared_buffer_data() + v_offset,
+         static_cast<const void*>(frame.data(VideoFrame::kVPlane)), v_size);
+
+  // TODO(xingliu): Maybe also copy the alpha plane in
+  // |MojoSharedBufferVideoFrame|. The alpha plane is ignored here, but
+  // the |shared_memory| should contain the space for alpha plane.
+
+  return mojo_frame;
+}
+
 // static
 scoped_refptr<MojoSharedBufferVideoFrame> MojoSharedBufferVideoFrame::Create(
     VideoPixelFormat format,

media/mojo/common/mojo_shared_buffer_video_frame.h

@@ -39,6 +39,12 @@ class MojoSharedBufferVideoFrame : public VideoFrame {
       const gfx::Size& dimensions,
       base::TimeDelta timestamp);
 
+  // Creates a YUV frame backed by shared memory from in-memory YUV frame.
+  // Internally the data from in-memory YUV frame will be copied to a
+  // consecutive block in shared memory. Will return null on failure.
+  static scoped_refptr<MojoSharedBufferVideoFrame> CreateFromYUVFrame(
+      const VideoFrame& frame);
+
   // Creates a MojoSharedBufferVideoFrame that uses the memory in |handle|.
   // This will take ownership of |handle|, so the caller can no longer use it.
   // |mojo_shared_buffer_done_cb|, if not null, is called on destruction,

media/mojo/common/mojo_shared_buffer_video_frame_unittest.cc

@@ -225,4 +225,28 @@ TEST(MojoSharedBufferVideoFrameTest, InterleavedData) {
   EXPECT_TRUE(frame->data(VideoFrame::kVPlane));
 }
 
+TEST(MojoSharedBufferVideoFrameTest, YUVFrameToMojoFrame) {
+  std::vector<uint8_t> data = std::vector<uint8_t>(12, 1u);
+  const auto pixel_format = VideoPixelFormat::PIXEL_FORMAT_I420;
+  const auto size = gfx::Size(1, 1);
+  scoped_refptr<VideoFrame> frame = VideoFrame::WrapExternalYuvData(
+      pixel_format, size, gfx::Rect(1, 1), size, 4, 4, 4, &data[0], &data[4],
+      &data[8], base::TimeDelta());
+  auto mojo_frame = MojoSharedBufferVideoFrame::CreateFromYUVFrame(*frame);
+  EXPECT_TRUE(mojo_frame);
+
+  const size_t u_offset =
+      VideoFrame::PlaneSize(pixel_format, VideoFrame::kYPlane, size).GetArea();
+  const size_t v_offset =
+      u_offset +
+      VideoFrame::PlaneSize(pixel_format, VideoFrame::kUPlane, size).GetArea();
+
+  // Verifies mapped size and offset.
+  EXPECT_EQ(mojo_frame->MappedSize(),
+            frame->AllocationSize(pixel_format, size));
+  EXPECT_EQ(mojo_frame->PlaneOffset(VideoFrame::kYPlane), 0u);
+  EXPECT_EQ(mojo_frame->PlaneOffset(VideoFrame::kUPlane), u_offset);
+  EXPECT_EQ(mojo_frame->PlaneOffset(VideoFrame::kVPlane), v_offset);
+}
+
 }  // namespace media