Media Galleries API: Audio/Video attached pictures support.

This patch enables audio / video thumbnail extraction. A lot of supporting code has already gone in. This is the last piece in the actual extension API that enables this.

BUG=304290

Review URL: https://codereview.chromium.org/250143002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@274849 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/extensions/api/media_galleries/media_galleries_api.cc

@@ -14,6 +14,7 @@
 #include "apps/app_window_registry.h"
 #include "base/callback.h"
 #include "base/lazy_instance.h"
+#include "base/numerics/safe_conversions.h"
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/utf_string_conversions.h"
@@ -36,11 +37,14 @@
 #include "chrome/common/pref_names.h"
 #include "components/storage_monitor/storage_info.h"
 #include "components/web_modal/web_contents_modal_dialog_manager.h"
+#include "content/public/browser/blob_handle.h"
+#include "content/public/browser/browser_context.h"
 #include "content/public/browser/browser_thread.h"
 #include "content/public/browser/child_process_security_policy.h"
 #include "content/public/browser/render_process_host.h"
 #include "content/public/browser/render_view_host.h"
 #include "content/public/browser/web_contents.h"
+#include "extensions/browser/blob_holder.h"
 #include "extensions/browser/event_router.h"
 #include "extensions/browser/extension_prefs.h"
 #include "extensions/browser/extension_system.h"
@@ -51,6 +55,7 @@
 #include "grit/generated_resources.h"
 #include "net/base/mime_sniffer.h"
 #include "ui/base/l10n/l10n_util.h"
+#include "webkit/browser/blob/blob_data_handle.h"
 
 using content::WebContents;
 using storage_monitor::MediaStorageUtil;
@@ -82,6 +87,12 @@ const char kIsMediaDeviceKey[] = "isMediaDevice";
 const char kIsRemovableKey[] = "isRemovable";
 const char kNameKey[] = "name";
 
+const char kMetadataKey[] = "metadata";
+const char kAttachedImagesBlobInfoKey[] = "attachedImagesBlobInfo";
+const char kBlobUUIDKey[] = "blobUUID";
+const char kTypeKey[] = "type";
+const char kSizeKey[] = "size";
+
 MediaFileSystemRegistry* media_file_system_registry() {
   return g_browser_process->media_file_system_registry();
 }
@@ -821,30 +832,28 @@ bool MediaGalleriesGetMetadataFunction::RunAsync() {
   if (!options)
     return false;
 
-  bool mime_type_only = options->metadata_type ==
-      MediaGalleries::GET_METADATA_TYPE_MIMETYPEONLY;
-
   return Setup(GetProfile(), &error_, base::Bind(
       &MediaGalleriesGetMetadataFunction::OnPreferencesInit, this,
-      mime_type_only, blob_uuid));
+      options->metadata_type, blob_uuid));
 }
 
 void MediaGalleriesGetMetadataFunction::OnPreferencesInit(
-    bool mime_type_only, const std::string& blob_uuid) {
+    MediaGalleries::GetMetadataType metadata_type,
+    const std::string& blob_uuid) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 
   // BlobReader is self-deleting.
   BlobReader* reader = new BlobReader(
       GetProfile(),
       blob_uuid,
-      base::Bind(&MediaGalleriesGetMetadataFunction::SniffMimeType, this,
-                 mime_type_only, blob_uuid));
+      base::Bind(&MediaGalleriesGetMetadataFunction::GetMetadata, this,
+                 metadata_type, blob_uuid));
   reader->SetByteRange(0, net::kMaxBytesToSniff);
   reader->Start();
 }
 
-void MediaGalleriesGetMetadataFunction::SniffMimeType(
-    bool mime_type_only, const std::string& blob_uuid,
+void MediaGalleriesGetMetadataFunction::GetMetadata(
+    MediaGalleries::GetMetadataType metadata_type, const std::string& blob_uuid,
     scoped_ptr<std::string> blob_header, int64 total_blob_length) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
 
@@ -857,19 +866,27 @@ void MediaGalleriesGetMetadataFunction::SniffMimeType(
     return;
   }
 
-  if (mime_type_only) {
+  if (metadata_type == MediaGalleries::GET_METADATA_TYPE_MIMETYPEONLY) {
     MediaGalleries::MediaMetadata metadata;
     metadata.mime_type = mime_type;
-    SetResult(metadata.ToValue().release());
+
+    base::DictionaryValue* result_dictionary = new base::DictionaryValue;
+    result_dictionary->Set(kMetadataKey, metadata.ToValue().release());
+    SetResult(result_dictionary);
     SendResponse(true);
     return;
   }
 
-  // TODO(tommycli): Enable getting attached images.
+  // We get attached images by default. GET_METADATA_TYPE_NONE is the default
+  // value if the caller doesn't specify the metadata type.
+  bool get_attached_images =
+      metadata_type == MediaGalleries::GET_METADATA_TYPE_ALL ||
+      metadata_type == MediaGalleries::GET_METADATA_TYPE_NONE;
+
   scoped_refptr<metadata::SafeMediaMetadataParser> parser(
       new metadata::SafeMediaMetadataParser(GetProfile(), blob_uuid,
                                             total_blob_length, mime_type,
-                                            false /* get_attached_images */));
+                                            get_attached_images));
   parser->Start(base::Bind(
       &MediaGalleriesGetMetadataFunction::OnSafeMediaMetadataParserDone, this));
 }
@@ -877,12 +894,94 @@ void MediaGalleriesGetMetadataFunction::SniffMimeType(
 void MediaGalleriesGetMetadataFunction::OnSafeMediaMetadataParserDone(
     bool parse_success, scoped_ptr<base::DictionaryValue> metadata_dictionary,
     scoped_ptr<std::vector<metadata::AttachedImage> > attached_images) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
   if (!parse_success) {
     SendResponse(false);
     return;
   }
 
-  SetResult(metadata_dictionary->DeepCopy());
+  DCHECK(metadata_dictionary.get());
+  DCHECK(attached_images.get());
+
+  scoped_ptr<base::DictionaryValue> result_dictionary(
+      new base::DictionaryValue);
+  result_dictionary->Set(kMetadataKey, metadata_dictionary.release());
+
+  if (attached_images->empty()) {
+    SetResult(result_dictionary.release());
+    SendResponse(true);
+    return;
+  }
+
+  result_dictionary->Set(kAttachedImagesBlobInfoKey, new base::ListValue);
+  metadata::AttachedImage* first_image = &attached_images->front();
+  content::BrowserContext::CreateMemoryBackedBlob(
+      GetProfile(),
+      first_image->data.c_str(),
+      first_image->data.size(),
+      base::Bind(&MediaGalleriesGetMetadataFunction::ConstructNextBlob,
+                 this, base::Passed(&result_dictionary),
+                 base::Passed(&attached_images),
+                 base::Passed(make_scoped_ptr(new std::vector<std::string>))));
+}
+
+void MediaGalleriesGetMetadataFunction::ConstructNextBlob(
+    scoped_ptr<base::DictionaryValue> result_dictionary,
+    scoped_ptr<std::vector<metadata::AttachedImage> > attached_images,
+    scoped_ptr<std::vector<std::string> > blob_uuids,
+    scoped_ptr<content::BlobHandle> current_blob) {
+  DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
+
+  DCHECK(result_dictionary.get());
+  DCHECK(attached_images.get());
+  DCHECK(blob_uuids.get());
+  DCHECK(current_blob.get());
+
+  DCHECK(!attached_images->empty());
+  DCHECK_LT(blob_uuids->size(), attached_images->size());
+
+  // For the newly constructed Blob, store its image's metadata and Blob UUID.
+  base::ListValue* attached_images_list = NULL;
+  result_dictionary->GetList(kAttachedImagesBlobInfoKey, &attached_images_list);
+  DCHECK(attached_images_list);
+  DCHECK_LT(attached_images_list->GetSize(), attached_images->size());
+
+  metadata::AttachedImage* current_image =
+      &(*attached_images)[blob_uuids->size()];
+  base::DictionaryValue* attached_image = new base::DictionaryValue;
+  attached_image->Set(kBlobUUIDKey, new base::StringValue(
+      current_blob->GetUUID()));
+  attached_image->Set(kTypeKey, new base::StringValue(
+      current_image->type));
+  attached_image->Set(kSizeKey, new base::FundamentalValue(
+      base::checked_cast<int>(current_image->data.size())));
+  attached_images_list->Append(attached_image);
+
+  blob_uuids->push_back(current_blob->GetUUID());
+  WebContents* contents = WebContents::FromRenderViewHost(render_view_host());
+  extensions::BlobHolder* holder =
+      extensions::BlobHolder::FromRenderProcessHost(
+          contents->GetRenderProcessHost());
+  holder->HoldBlobReference(current_blob.Pass());
+
+  // Construct the next Blob if necessary.
+  if (blob_uuids->size() < attached_images->size()) {
+    metadata::AttachedImage* next_image =
+        &(*attached_images)[blob_uuids->size()];
+    content::BrowserContext::CreateMemoryBackedBlob(
+        GetProfile(),
+        next_image->data.c_str(),
+        next_image->data.size(),
+        base::Bind(&MediaGalleriesGetMetadataFunction::ConstructNextBlob,
+                   this, base::Passed(&result_dictionary),
+                   base::Passed(&attached_images), base::Passed(&blob_uuids)));
+    return;
+  }
+
+  // All Blobs have been constructed. The renderer will take ownership.
+  SetResult(result_dictionary.release());
+  SetTransferredBlobUUIDs(*blob_uuids);
   SendResponse(true);
 }
 

chrome/browser/extensions/api/media_galleries/media_galleries_api.h

@@ -27,6 +27,7 @@ namespace MediaGalleries = extensions::api::media_galleries;
 class MediaGalleriesScanResultDialogController;
 
 namespace content {
+class BlobHandle;
 class WebContents;
 }
 
@@ -264,16 +265,23 @@ class MediaGalleriesGetMetadataFunction : public ChromeAsyncExtensionFunction {
 
  private:
   // Bottom half for RunAsync, invoked after the preferences is initialized.
-  void OnPreferencesInit(bool mime_type_only, const std::string& blob_uuid);
+  void OnPreferencesInit(MediaGalleries::GetMetadataType metadata_type,
+                         const std::string& blob_uuid);
 
-  void SniffMimeType(bool mime_type_only,
-                     const std::string& blob_uuid,
-                     scoped_ptr<std::string> blob_header,
-                     int64 total_blob_length);
+  void GetMetadata(MediaGalleries::GetMetadataType metadata_type,
+                   const std::string& blob_uuid,
+                   scoped_ptr<std::string> blob_header,
+                   int64 total_blob_length);
 
   void OnSafeMediaMetadataParserDone(
-      bool parse_success, scoped_ptr<base::DictionaryValue> metadata_dictionary,
+      bool parse_success, scoped_ptr<base::DictionaryValue> result_dictionary,
       scoped_ptr<std::vector<metadata::AttachedImage> > attached_images);
+
+  void ConstructNextBlob(
+      scoped_ptr<base::DictionaryValue> result_dictionary,
+      scoped_ptr<std::vector<metadata::AttachedImage> > attached_images,
+      scoped_ptr<std::vector<std::string> > blob_uuids,
+      scoped_ptr<content::BlobHandle> current_blob);
 };
 
 }  // namespace extensions

chrome/common/extensions/api/media_galleries.idl

@@ -17,8 +17,10 @@ namespace mediaGalleries {
   };
 
   [inline_doc] enum GetMetadataType {
-    // Retrieve all available metadata.
+    // Retrieve the mime type, metadata tags, and attached images.
     all,
+    // Retrieve only the mime type and the metadata tags.
+    mimeTypeAndTags,
     // Retrieve only the mime type.
     mimeTypeOnly
   };
@@ -147,6 +149,10 @@ namespace mediaGalleries {
     // All the metadata in the media file. For formats with multiple streams,
     // stream order will be preserved. Container metadata is the first element.
     StreamInfo[] rawTags;
+
+    // The images embedded in the media file's metadata. This is most often
+    // used for album art or video thumbnails.
+    [instanceof=Blob] object[] attachedImages;
   };
 
   callback MediaMetadataCallback = void (MediaMetadata metadata);

chrome/renderer/resources/extensions/media_galleries_custom_bindings.js

@@ -103,6 +103,22 @@ binding.registerCustomHook(function(bindingsAPI, extensionId) {
     var blobUuid = blobNatives.GetBlobUuid(mediaFile)
     return [blobUuid, options, callback];
   });
+
+  apiFunctions.setCustomCallback('getMetadata',
+                                 function(name, request, response) {
+    if (response.attachedImagesBlobInfo) {
+      for (var i = 0; i < response.attachedImagesBlobInfo.length; i++) {
+        var blobInfo = response.attachedImagesBlobInfo[i];
+        var blob = blobNatives.TakeBrowserProcessBlob(
+            blobInfo.blobUUID, blobInfo.type, blobInfo.size);
+        response.metadata.attachedImages.push(blob);
+      }
+    }
+
+    if (request.callback)
+      request.callback(response.metadata);
+    request.callback = null;
+  });
 });
 
 exports.binding = binding.generate();

chrome/test/data/extensions/api_test/media_galleries/media_metadata/test.js

@@ -26,6 +26,9 @@ function RunMetadataTest(filename, callOptions, verifyMetadataFunction) {
 function ImageMIMETypeOnlyTest() {
   function verifyMetadata(metadata) {
     chrome.test.assertEq("image/jpeg", metadata.mimeType);
+
+    chrome.test.assertEq(0, metadata.attachedImages.length);
+
     chrome.test.succeed();
   }
 
@@ -47,6 +50,9 @@ function ImageTagsTest() {
     chrome.test.assertEq(3.2, metadata.fNumber);
     chrome.test.assertEq(100, metadata.focalLengthMm);
     chrome.test.assertEq(1600, metadata.isoEquivalent);
+
+    chrome.test.assertEq(0, metadata.attachedImages.length);
+
     chrome.test.succeed();
   }
 
@@ -57,6 +63,9 @@ function MP3MIMETypeOnlyTest() {
   function verifyMetadata(metadata) {
     chrome.test.assertEq("audio/mpeg", metadata.mimeType);
     chrome.test.assertEq(undefined, metadata.title);
+
+    chrome.test.assertEq(0, metadata.attachedImages.length);
+
     chrome.test.succeed();
   }
 
@@ -88,9 +97,42 @@ function MP3TagsTest() {
 
     chrome.test.assertEq("png", metadata.rawTags[2].type);
 
+    chrome.test.assertEq(0, metadata.attachedImages.length);
+
     chrome.test.succeed();
   }
 
+  return RunMetadataTest("id3_png_test.mp3", {metadataType: 'mimeTypeAndTags'},
+                         verifyMetadata);
+}
+
+function MP3AttachedImageTest() {
+  function verifyMetadata(metadata) {
+    chrome.test.assertEq("audio/mpeg", metadata.mimeType);
+    chrome.test.assertEq("Airbag", metadata.title);
+    chrome.test.assertEq("Radiohead", metadata.artist);
+    chrome.test.assertEq("OK Computer", metadata.album);
+    chrome.test.assertEq(1, metadata.track);
+    chrome.test.assertEq("Alternative", metadata.genre);
+
+    chrome.test.assertEq(1, metadata.attachedImages.length);
+    chrome.test.assertEq('image/png', metadata.attachedImages[0].type);
+    chrome.test.assertEq(155752, metadata.attachedImages[0].size);
+
+    var reader = new FileReader();
+    reader.onload = function verifyBlobContents(event) {
+      var first = new Uint8Array(reader.result, 0, 8);
+      var last = new Uint8Array(reader.result, reader.result.byteLength - 8, 8);
+      chrome.test.assertEq("\x89PNG\r\n\x1a\n",
+                           String.fromCharCode.apply(null, first));
+      chrome.test.assertEq("IEND\xae\x42\x60\x82",
+                           String.fromCharCode.apply(null, last));
+
+      chrome.test.succeed();
+    }
+    reader.readAsArrayBuffer(metadata.attachedImages[0]);
+  }
+
   return RunMetadataTest("id3_png_test.mp3", {}, verifyMetadata);
 }
 
@@ -124,6 +166,8 @@ function RotatedVideoTest() {
                          metadata.rawTags[2].tags["handler_name"]);
     chrome.test.assertEq("eng", metadata.rawTags[2].tags["language"]);
 
+    chrome.test.assertEq(0, metadata.attachedImages.length);
+
     chrome.test.succeed();
   }
 
@@ -144,6 +188,7 @@ chrome.test.getConfig(function(config) {
     testsToRun = testsToRun.concat([
       MP3MIMETypeOnlyTest,
       MP3TagsTest,
+      MP3AttachedImageTest,
       RotatedVideoTest
     ]);
   }

extensions/browser/blob_holder.cc

@@ -44,8 +44,8 @@ void BlobHolder::HoldBlobReference(scoped_ptr<content::BlobHandle> blob) {
   DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
   DCHECK(!ContainsBlobHandle(blob.get()));
 
-  held_blobs_.insert(
-      make_pair(blob->GetUUID(), make_linked_ptr(blob.release())));
+  std::string uuid = blob->GetUUID();
+  held_blobs_.insert(make_pair(uuid, make_linked_ptr(blob.release())));
 }
 
 BlobHolder::BlobHolder(content::RenderProcessHost* render_process_host)