Android Video Thumbnail: Render video frame to bitmap.

This CL finishes the video thumbnail pipeline for most codec except
vp8, vp9. Now it's ready to hook to the UI frontend. It does following:

1. Render the video frame with existing media toolkit, which can read
back pixel data from a texture in GPU process.

2. Plumbs a few metadata fields and thumbnail bitmap to Java layer.

Bug: 826021
Change-Id: Id6635975726f4b12ab51e01cb6b3e78a198632f4
Reviewed-on: https://chromium-review.googlesource.com/1222694Reviewed-by: Min Qin <qinmin@chromium.org>
Reviewed-by: David Trainor <dtrainor@chromium.org>
Commit-Queue: Xing Liu <xingliu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#591406}

chrome/android/java/src/org/chromium/chrome/browser/download/DownloadMediaData.java

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.chrome.browser.download;
+
+import android.graphics.Bitmap;
+
+import org.chromium.base.annotations.CalledByNative;
+
+/**
+ * Contains local media metadata and thumbnails.
+ */
+public class DownloadMediaData {
+    /**
+     * The duration of the media file in seconds. Or -1 if no duration in the metadata.
+     */
+    public final double duration;
+
+    /**
+     * Title of the media.
+     */
+    public final String title;
+
+    /**
+     * Artist of the media.
+     */
+    public final String artist;
+
+    /**
+     * A thumbnail represents the media. Can be a poster image retrieved with metadata for both
+     * audio and video file. Or retrieved from a video key frame for video files.
+     */
+    public final Bitmap thumbnail;
+
+    @CalledByNative
+    private DownloadMediaData(double duration, String title, String artist, Bitmap thumbnail) {
+        this.duration = duration;
+        this.title = title;
+        this.artist = artist;
+        this.thumbnail = thumbnail;
+    }
+}

chrome/android/java/src/org/chromium/chrome/browser/download/DownloadMediaParserBridge.java

@@ -19,8 +19,8 @@ public class DownloadMediaParserBridge {
      * @param totalSize Total size of the media file.
      * @param callback Callback to get the result.
      */
-    public DownloadMediaParserBridge(
-            String mimeType, String filePath, long totalSize, Callback<Boolean> callback) {
+    public DownloadMediaParserBridge(String mimeType, String filePath, long totalSize,
+            Callback<DownloadMediaData> callback) {
         mNativeDownloadMediaParserBridge = nativeInit(mimeType, filePath, totalSize, callback);
     }
 
@@ -43,7 +43,7 @@ public class DownloadMediaParserBridge {
     }
 
     private native long nativeInit(
-            String mimeType, String filePath, long totalSize, Callback<Boolean> callback);
+            String mimeType, String filePath, long totalSize, Callback<DownloadMediaData> callback);
     private native void nativeDestory(long nativeDownloadMediaParserBridge);
     private native void nativeStart(long nativeDownloadMediaParserBridge);
 }

chrome/android/java_sources.gni

@@ -440,6 +440,7 @@ chrome_java_sources = [
   "java/src/org/chromium/chrome/browser/download/DownloadLocationDialogBridge.java",
   "java/src/org/chromium/chrome/browser/download/DownloadManagerDelegate.java",
   "java/src/org/chromium/chrome/browser/download/DownloadManagerService.java",
+  "java/src/org/chromium/chrome/browser/download/DownloadMediaData.java",
   "java/src/org/chromium/chrome/browser/download/DownloadMediaParserBridge.java",
   "java/src/org/chromium/chrome/browser/download/DownloadMetrics.java",
   "java/src/org/chromium/chrome/browser/download/DownloadNotificationFactory.java",

chrome/android/javatests/src/org/chromium/chrome/browser/download/DownloadMediaParserTest.java

@@ -46,7 +46,7 @@ public class DownloadMediaParserTest {
      */
     public static class MediaParseResult {
         public boolean done;
-        public boolean success;
+        public DownloadMediaData mediaData;
     }
 
     @Before
@@ -68,8 +68,8 @@ public class DownloadMediaParserTest {
         // The native DownloadMediaParser needs to be created on UI thread.
         ThreadUtils.runOnUiThreadBlocking(() -> {
             DownloadMediaParserBridge parser = new DownloadMediaParserBridge(
-                    "audio/mp3", filePath, audioFile.length(), (success) -> {
-                        result.success = success;
+                    "audio/mp3", filePath, audioFile.length(), (DownloadMediaData mediaData) -> {
+                        result.mediaData = mediaData;
                         result.done = true;
                     });
             parser.start();
@@ -82,7 +82,7 @@ public class DownloadMediaParserTest {
             }
         }, MAX_MEDIA_PARSER_POLL_TIME_MS, MEDIA_PARSER_POLL_INTERVAL_MS);
 
-        Assert.assertTrue("Failed to parse audio metadata.", result.success);
+        Assert.assertTrue("Failed to parse audio metadata.", result.mediaData != null);
     }
 
     @Test
@@ -99,8 +99,8 @@ public class DownloadMediaParserTest {
         // The native DownloadMediaParser needs to be created on UI thread.
         ThreadUtils.runOnUiThreadBlocking(() -> {
             DownloadMediaParserBridge parser = new DownloadMediaParserBridge(
-                    "video/mp4", filePath, videoFile.length(), (success) -> {
-                        result.success = success;
+                    "video/mp4", filePath, videoFile.length(), (DownloadMediaData mediaData) -> {
+                        result.mediaData = mediaData;
                         result.done = true;
                     });
             parser.start();
@@ -113,6 +113,10 @@ public class DownloadMediaParserTest {
             }
         }, MAX_MEDIA_PARSER_POLL_TIME_MS, MEDIA_PARSER_POLL_INTERVAL_MS);
 
-        Assert.assertTrue("Failed to parse video file.", result.success);
+        Assert.assertTrue("Failed to parse video file.", result.mediaData != null);
+        Assert.assertTrue(
+                "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getWidth() > 0);
+        Assert.assertTrue(
+                "Failed to retrieve thumbnail.", result.mediaData.thumbnail.getHeight() > 0);
     }
 }

chrome/browser/BUILD.gn

@@ -2125,6 +2125,8 @@ jumbo_split_static_library("browser") {
       "android/download/service/download_background_task.cc",
       "android/download/service/download_task_scheduler.cc",
       "android/download/service/download_task_scheduler.h",
+      "android/download/video_frame_thumbnail_converter.cc",
+      "android/download/video_frame_thumbnail_converter.h",
       "android/explore_sites/catalog.cc",
       "android/explore_sites/catalog.h",
       "android/explore_sites/explore_sites_bridge.cc",
@@ -4570,6 +4572,7 @@ if (is_android) {
       "../android/java/src/org/chromium/chrome/browser/download/DownloadItem.java",
       "../android/java/src/org/chromium/chrome/browser/download/DownloadLocationDialogBridge.java",
       "../android/java/src/org/chromium/chrome/browser/download/DownloadManagerService.java",
+      "../android/java/src/org/chromium/chrome/browser/download/DownloadMediaData.java",
       "../android/java/src/org/chromium/chrome/browser/download/DownloadMediaParserBridge.java",
       "../android/java/src/org/chromium/chrome/browser/download/DownloadUtils.java",
       "../android/java/src/org/chromium/chrome/browser/download/items/OfflineContentAggregatorFactory.java",

chrome/browser/android/download/download_media_parser.cc

@@ -10,6 +10,7 @@
 #include "base/task/post_task.h"
 #include "base/task/task_traits.h"
 #include "chrome/browser/android/download/local_media_data_source_factory.h"
+#include "chrome/browser/android/download/video_frame_thumbnail_converter.h"
 #include "content/public/browser/android/gpu_video_accelerator_factories_provider.h"
 #include "content/public/common/service_manager_connection.h"
 #include "media/base/overlay_info.h"
@@ -19,6 +20,8 @@
 #include "media/mojo/services/media_interface_provider.h"
 #include "media/video/gpu_video_accelerator_factories.h"
 #include "services/service_manager/public/cpp/connector.h"
+#include "services/ws/public/cpp/gpu/context_provider_command_buffer.h"
+#include "third_party/skia/include/core/SkBitmap.h"
 
 namespace {
 
@@ -91,6 +94,7 @@ void DownloadMediaParser::OnMediaMetadataParsed(
     return;
   }
   metadata_ = std::move(metadata);
+  DCHECK(metadata_);
 
   // TODO(xingliu): Make |attached_images| movable and use this as a thumbnail
   // source as well as video frame.
@@ -99,14 +103,14 @@ void DownloadMediaParser::OnMediaMetadataParsed(
   // For audio file, we only need metadata and poster.
   if (base::StartsWith(mime_type_, "audio/",
                        base::CompareCase::INSENSITIVE_ASCII)) {
-    NotifyComplete();
+    NotifyComplete(SkBitmap());
     return;
   }
 
   DCHECK(base::StartsWith(mime_type_, "video/",
                           base::CompareCase::INSENSITIVE_ASCII));
 
-  // Retrieves video thumbnail if needed.
+  // Start to retrieve video thumbnail.
   base::ThreadTaskRunnerHandle::Get()->PostTask(
       FROM_HERE, base::BindOnce(&DownloadMediaParser::RetrieveEncodedVideoFrame,
                                 weak_factory_.GetWeakPtr()));
@@ -213,9 +217,29 @@ void DownloadMediaParser::OnVideoFrameDecoded(
   DCHECK(frame->HasTextures());
   decode_done_ = true;
 
-  // TODO(xingliu): render the |frame| in the browser process, with cc or skia
-  // or gl. The |frame| is associated with a native texture in GPU process.
-  NotifyComplete();
+  RenderVideoFrame(frame);
+}
+
+void DownloadMediaParser::RenderVideoFrame(
+    const scoped_refptr<media::VideoFrame>& video_frame) {
+  auto converter = VideoFrameThumbnailConverter::Create(
+      config_.codec(), gpu_factories_->GetMediaContextProvider());
+  converter->ConvertToBitmap(
+      video_frame,
+      base::BindOnce(&DownloadMediaParser::OnBitmapGenerated,
+                     weak_factory_.GetWeakPtr(), std::move(converter)));
+}
+
+void DownloadMediaParser::OnBitmapGenerated(
+    std::unique_ptr<VideoFrameThumbnailConverter>,
+    bool success,
+    SkBitmap bitmap) {
+  if (!success) {
+    OnError();
+    return;
+  }
+
+  NotifyComplete(std::move(bitmap));
 }
 
 media::mojom::InterfaceFactory*
@@ -250,14 +274,17 @@ void DownloadMediaParser::OnMediaDataReady(
     std::move(callback).Run(std::vector<uint8_t>(data->begin(), data->end()));
 }
 
-void DownloadMediaParser::NotifyComplete() {
+void DownloadMediaParser::NotifyComplete(SkBitmap bitmap) {
   // TODO(xingliu): Return the metadata and video thumbnail data in
   // |parse_complete_cb_|.
+  DCHECK(metadata_);
   if (parse_complete_cb_)
-    std::move(parse_complete_cb_).Run(true);
+    std::move(parse_complete_cb_)
+        .Run(true, std::move(metadata_), std::move(bitmap));
 }
 
 void DownloadMediaParser::OnError() {
   if (parse_complete_cb_)
-    std::move(parse_complete_cb_).Run(false);
+    std::move(parse_complete_cb_)
+        .Run(false, chrome::mojom::MediaMetadata::New(), SkBitmap());
 }

chrome/browser/android/download/download_media_parser.h

@@ -28,11 +28,21 @@ class MojoVideoDecoder;
 class VideoDecoderConfig;
 }  // namespace media
 
-// Local media files parser is used to process local media files. This object
-// lives on main thread in browser process.
+class SkBitmap;
+class VideoFrameThumbnailConverter;
+
+// Parse local media files, including media metadata and thumbnails.
+// Metadata is always parsed in utility process for both audio and video files.
+//
+// For video file, the thumbnail may be poster image in metadata or extracted
+// video frame. The frame extraction always happens in utility process. The
+// decoding may happen in utility or GPU process based on video codec.
 class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
  public:
-  using ParseCompleteCB = base::OnceCallback<void(bool)>;
+  using ParseCompleteCB =
+      base::OnceCallback<void(bool success,
+                              chrome::mojom::MediaMetadataPtr media_metadata,
+                              SkBitmap bitmap)>;
 
   DownloadMediaParser(int64_t size,
                       const std::string& mime_type,
@@ -42,8 +52,8 @@ class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
 
   // Parse media metadata in a local file. All file IO will run on
   // |file_task_runner|. The metadata is parsed in an utility process safely.
-  // However, the result is still comes from user-defined input, thus should be
-  // used with caution.
+  // The thumbnail is retrieved from GPU process or utility process based on
+  // different codec.
   void Start();
 
  private:
@@ -75,12 +85,18 @@ class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
   media::mojom::InterfaceFactory* GetMediaInterfaceFactory();
   void OnDecoderConnectionError();
 
+  // Renders the video frame to bitmap.
+  void RenderVideoFrame(const scoped_refptr<media::VideoFrame>& video_frame);
+  void OnBitmapGenerated(std::unique_ptr<VideoFrameThumbnailConverter>,
+                         bool success,
+                         SkBitmap bitmap);
+
   // Overlays media data source read operation. Gradually read data from media
   // file.
   void OnMediaDataReady(chrome::mojom::MediaDataSource::ReadCallback callback,
                         std::unique_ptr<std::string> data);
 
-  void NotifyComplete();
+  void NotifyComplete(SkBitmap bitmap);
   void OnError();
 
   int64_t size_;
@@ -89,6 +105,8 @@ class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
 
   ParseCompleteCB parse_complete_cb_;
   chrome::mojom::MediaMetadataPtr metadata_;
+
+  // Poster images obtained with |metadata_|.
   std::vector<metadata::AttachedImage> attached_images_;
 
   std::unique_ptr<chrome::mojom::MediaDataSource> media_data_source_;
@@ -96,9 +114,8 @@ class DownloadMediaParser : public MediaParserProvider, public media::MediaLog {
   // The task runner to do blocking disk IO.
   scoped_refptr<base::SequencedTaskRunner> file_task_runner_;
 
-  // Encoded frame to be decoded. The data comes from a safe sandboxed process.
-  // This data can be large for high resolution video, should be std::move or
-  // cleared whenever possible.
+  // Encoded video frame to be decoded. This data can be large for high
+  // resolution video, should be std::move or cleared whenever possible.
   std::vector<uint8_t> encoded_data_;
 
   // Objects used to decode the video into media::VideoFrame.

chrome/browser/android/download/download_media_parser_bridge.cc

@@ -8,13 +8,35 @@
 #include "base/android/jni_string.h"
 #include "base/files/file_path.h"
 #include "base/task/post_task.h"
+#include "jni/DownloadMediaData_jni.h"
 #include "jni/DownloadMediaParserBridge_jni.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "ui/gfx/android/java_bitmap.h"
+
+using base::android::ConvertUTF8ToJavaString;
 
 namespace {
 
 void OnMediaParsed(const base::android::ScopedJavaGlobalRef<jobject> jcallback,
-                   bool success) {
-  base::android::RunBooleanCallbackAndroid(jcallback, success);
+                   bool success,
+                   chrome::mojom::MediaMetadataPtr metadata,
+                   SkBitmap thumbnail_bitmap) {
+  JNIEnv* env = base::android::AttachCurrentThread();
+  DCHECK(metadata);
+
+  // Copy the thumbnail bitmap to a Java Bitmap object.
+  base::android::ScopedJavaLocalRef<jobject> java_bitmap;
+  if (!thumbnail_bitmap.isNull())
+    java_bitmap = gfx::ConvertToJavaBitmap(&thumbnail_bitmap);
+
+  base::android::ScopedJavaLocalRef<jobject> media_data;
+  if (success) {
+    media_data = Java_DownloadMediaData_Constructor(
+        env, metadata->duration, ConvertUTF8ToJavaString(env, metadata->title),
+        ConvertUTF8ToJavaString(env, metadata->artist), std::move(java_bitmap));
+  }
+
+  base::android::RunObjectCallbackAndroid(jcallback, std::move(media_data));
 }
 
 }  // namespace

chrome/browser/android/download/video_frame_thumbnail_converter.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/android/download/video_frame_thumbnail_converter.h"
+
+#include "base/macros.h"
+#include "cc/paint/skia_paint_canvas.h"
+#include "components/viz/common/gl_helper.h"
+#include "media/base/video_frame.h"
+#include "media/renderers/paint_canvas_video_renderer.h"
+#include "services/ws/public/cpp/gpu/context_provider_command_buffer.h"
+
+namespace {
+
+// Class to generate bitmap based on video frame backed by texture in remote
+// process.
+class TextureFrameThumbnailConverter : public VideoFrameThumbnailConverter {
+ public:
+  TextureFrameThumbnailConverter(
+      scoped_refptr<ws::ContextProviderCommandBuffer> context_provider)
+      : context_provider_(std::move(context_provider)) {}
+  TextureFrameThumbnailConverter() = default;
+  ~TextureFrameThumbnailConverter() override = default;
+
+ private:
+  // VideoFrameThumbnailConverter implementation.
+  void ConvertToBitmap(const scoped_refptr<media::VideoFrame>& frame,
+                       BitmapCallback pixel_callback) override {
+    DCHECK(frame->HasTextures())
+        << "This implementation only do texture read backs.";
+
+    // Read back the texture data contained in the video frame.
+    media::PaintCanvasVideoRenderer renderer;
+    SkBitmap skbitmap;
+    skbitmap.allocN32Pixels(frame->visible_rect().width(),
+                            frame->visible_rect().height());
+    cc::SkiaPaintCanvas canvas(skbitmap);
+    renderer.Copy(frame, &canvas,
+                  media::Context3D(context_provider_->ContextGL(),
+                                   context_provider_->GrContext()));
+
+    std::move(pixel_callback).Run(true, std::move(skbitmap));
+  }
+
+  // Command buffer channel used to read back the pixel data from texture in
+  // remote process.
+  scoped_refptr<ws::ContextProviderCommandBuffer> context_provider_;
+
+  DISALLOW_COPY_AND_ASSIGN(TextureFrameThumbnailConverter);
+};
+
+}  // namespace
+
+// static
+std::unique_ptr<VideoFrameThumbnailConverter>
+VideoFrameThumbnailConverter::Create(
+    media::VideoCodec codec,
+    scoped_refptr<ws::ContextProviderCommandBuffer> context_provider) {
+  // TODO(xingliu): Implement vpx decode and render pipeline.
+  DCHECK_NE(codec, media::VideoCodec::kCodecVP8);
+  DCHECK_NE(codec, media::VideoCodec::kCodecVP9);
+
+  return std::make_unique<TextureFrameThumbnailConverter>(
+      std::move(context_provider));
+}

chrome/browser/android/download/video_frame_thumbnail_converter.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_ANDROID_DOWNLOAD_VIDEO_FRAME_THUMBNAIL_CONVERTER_H_
+#define CHROME_BROWSER_ANDROID_DOWNLOAD_VIDEO_FRAME_THUMBNAIL_CONVERTER_H_
+
+#include <memory>
+#include <vector>
+
+#include "base/callback.h"
+#include "base/memory/scoped_refptr.h"
+#include "media/base/video_codecs.h"
+
+namespace media {
+class VideoFrame;
+}  // namespace media
+
+namespace ws {
+class ContextProviderCommandBuffer;
+}  // namespace ws
+
+class SkBitmap;
+
+// Generates video thumbnail from a video frame. For most video format such as
+// H264, the input video frame contains a texture created in GPU process, we do
+// a readback and generate the bitmap. For vp8, vp9 format, the video frame
+// contains in-memory YUV data.
+class VideoFrameThumbnailConverter {
+ public:
+  using BitmapCallback = base::OnceCallback<void(bool, SkBitmap)>;
+
+  // Create the video thumbnail renderer for video type with |codec|. The
+  // decoder to generate video frame may yields video frames backed by texture
+  // in remote process or raw YUV data.
+  static std::unique_ptr<VideoFrameThumbnailConverter> Create(
+      media::VideoCodec codec,
+      scoped_refptr<ws::ContextProviderCommandBuffer> context_provider);
+
+  virtual void ConvertToBitmap(const scoped_refptr<media::VideoFrame>& frame,
+                               BitmapCallback pixel_callback) = 0;
+
+  virtual ~VideoFrameThumbnailConverter() = default;
+};
+
+#endif  // CHROME_BROWSER_ANDROID_DOWNLOAD_VIDEO_FRAME_THUMBNAIL_CONVERTER_H_

content/browser/media/android/browser_gpu_video_accelerator_factories.cc

@@ -41,7 +41,7 @@ void OnGpuChannelEstablished(
 
   constexpr bool automatic_flushes = false;
   constexpr bool support_locking = false;
-  constexpr bool support_grcontext = false;
+  constexpr bool support_grcontext = true;
 
   auto context_provider =
       base::MakeRefCounted<ws::ContextProviderCommandBuffer>(
@@ -187,7 +187,7 @@ BrowserGpuVideoAcceleratorFactories::
 
 scoped_refptr<ws::ContextProviderCommandBuffer>
 BrowserGpuVideoAcceleratorFactories::GetMediaContextProvider() {
-  return nullptr;
+  return context_provider_;
 }
 
 void BrowserGpuVideoAcceleratorFactories::SetRenderingColorSpace(