Add new VideoFrameMetata CopyMode.

Remove |copy_required| and replace it with an enum |copy_mode|.

This is done since in future texture copy will not be always required
for webview and hence |copy_required| will not always be true.
When AImageReader is enabled, texture copy can be skipped
and resource sharing across different context can be achieved via
AHardwareBuffer.

TBR=dcheng@chromium.org

Bug: 1091945
Change-Id: Ie482546cc2f28128b487a55205a3c7411bc2a86f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2306670
Commit-Queue: vikas soni <vikassoni@chromium.org>
Reviewed-by: Khushal <khushalsagar@chromium.org>
Reviewed-by: Dale Curtis <dalecurtis@chromium.org>
Reviewed-by: Frank Liberato <liberato@chromium.org>
Cr-Commit-Position: refs/heads/master@{#792652}

content/renderer/media/android/stream_texture_wrapper_impl.cc

@@ -85,8 +85,10 @@ void StreamTextureWrapperImpl::CreateVideoFrame(
           coded_size, visible_rect, visible_rect.size(), base::TimeDelta());
   new_frame->set_ycbcr_info(ycbcr_info);
 
-  if (enable_texture_copy_)
-    new_frame->metadata()->copy_required = true;
+  if (enable_texture_copy_) {
+    new_frame->metadata()->copy_mode =
+        media::VideoFrameMetadata::CopyMode::kCopyToNewTexture;
+  }
 
   SetCurrentFrameInternal(new_frame);
 }

media/base/video_frame_metadata.cc

@@ -31,7 +31,7 @@ void VideoFrameMetadata::MergeMetadataFrom(
   MERGE_FIELD(capture_end_time, metadata_source);
   MERGE_FIELD(capture_counter, metadata_source);
   MERGE_FIELD(capture_update_rect, metadata_source);
-  MERGE_FIELD(copy_required, metadata_source);
+  MERGE_FIELD(copy_mode, metadata_source);
   MERGE_FIELD(end_of_stream, metadata_source);
   MERGE_FIELD(frame_duration, metadata_source);
   MERGE_FIELD(frame_rate, metadata_source);

media/base/video_frame_metadata.h

@@ -26,6 +26,25 @@ struct MEDIA_EXPORT VideoFrameMetadata {
 
   VideoFrameMetadata(const VideoFrameMetadata& other);
 
+  enum CopyMode {
+    // Indicates that mailbox created in one context, is also being used in a
+    // different context belonging to another share group and video frames are
+    // using SurfaceTexture to render frames.
+    // Textures generated from SurfaceTexture can't be shared between contexts
+    // of different share group and hence this frame must be copied to a new
+    // texture before use, rather than being used directly.
+    kCopyToNewTexture = 0,
+
+    // Indicates that mailbox created in one context, is also being used in a
+    // different context belonging to another share group and video frames are
+    // using AImageReader to render frames.
+    // AImageReader allows to render image data to AHardwareBuffer which can be
+    // shared between contexts of different share group. AHB from existing
+    // mailbox is wrapped into a new mailbox(AHB backed) which can then be used
+    // by another context.
+    kCopyMailboxesOnly = 1,
+  };
+
   // Merges internal values from |metadata_source|.
   void MergeMetadataFrom(const VideoFrameMetadata* metadata_source);
 
@@ -56,11 +75,9 @@ struct MEDIA_EXPORT VideoFrameMetadata {
   // fully contained within visible_rect().
   base::Optional<gfx::Rect> capture_update_rect;
 
-  // Indicates that this frame must be copied to a new texture before use,
-  // rather than being used directly. Specifically this is required for
-  // WebView because of limitations about sharing surface textures between GL
-  // contexts.
-  bool copy_required = false;
+  // If not null, it indicates how video frame mailbox should be copied to a
+  // new mailbox.
+  base::Optional<CopyMode> copy_mode;
 
   // Indicates if the current frame is the End of its current Stream.
   bool end_of_stream = false;

media/base/video_frame_unittest.cc

@@ -63,9 +63,11 @@ media::VideoFrameMetadata GetFullVideoFrameMetadata() {
   // media::VideoRotations
   metadata.rotation = media::VideoRotation::VIDEO_ROTATION_90;
 
+  // media::VideoFrameMetadata::CopyMode
+  metadata.copy_mode = media::VideoFrameMetadata::CopyMode::kCopyToNewTexture;
+
   // bools
   metadata.allow_overlay = true;
-  metadata.copy_required = true;
   metadata.end_of_stream = true;
   metadata.texture_owner = true;
   metadata.wants_promotion_hint = true;
@@ -112,7 +114,7 @@ void VerifyVideoFrameMetadataEquality(const media::VideoFrameMetadata& a,
   EXPECT_EQ(a.capture_end_time, b.capture_end_time);
   EXPECT_EQ(a.capture_counter, b.capture_counter);
   EXPECT_EQ(a.capture_update_rect, b.capture_update_rect);
-  EXPECT_EQ(a.copy_required, b.copy_required);
+  EXPECT_EQ(a.copy_mode, b.copy_mode);
   EXPECT_EQ(a.end_of_stream, b.end_of_stream);
   EXPECT_EQ(a.frame_duration, b.frame_duration);
   EXPECT_EQ(a.frame_rate, b.frame_rate);

media/gpu/android/video_frame_factory_impl.cc

@@ -285,8 +285,10 @@ void VideoFrameFactoryImpl::CreateVideoFrame_OnImageReady(
 
   // The frames must be copied when threaded texture mailboxes are in use
   // (http://crbug.com/582170).
-  if (enable_threaded_texture_mailboxes)
-    frame->metadata()->copy_required = true;
+  if (enable_threaded_texture_mailboxes) {
+    frame->metadata()->copy_mode =
+        VideoFrameMetadata::CopyMode::kCopyToNewTexture;
+  }
 
   const bool is_surface_control =
       overlay_mode == OverlayMode::kSurfaceControlSecure ||

media/mojo/mojom/BUILD.gn

@@ -144,6 +144,10 @@ mojom("mojom") {
           mojom = "media.mojom.VideoRotation"
           cpp = "::media::VideoRotation"
         },
+        {
+          mojom = "media.mojom.CopyMode"
+          cpp = "::media::VideoFrameMetadata::CopyMode"
+        },
       ]
       traits_headers = [ "media_types_enum_mojom_traits.h" ]
     },

media/mojo/mojom/media_types.mojom

@@ -73,6 +73,12 @@ enum VideoRotation {
   kVideoRotation270,
 };
 
+// see media/base/video_frame_metadata.h for descriptions.
+enum CopyMode {
+  kCopyToNewTexture,
+  kCopyMailboxesOnly,
+};
+
 // See media/base/video_transformation.h for descriptions.
 struct VideoTransformation {
   VideoRotation rotation;
@@ -273,7 +279,8 @@ struct VideoFrameMetadata {
 
   gfx.mojom.Rect? capture_update_rect;
 
-  bool copy_required;
+  bool has_copy_mode;
+  CopyMode copy_mode;
 
   bool end_of_stream;
 

media/mojo/mojom/media_types_enum_mojom_traits.h

@@ -6,6 +6,7 @@
 #define MEDIA_MOJO_MOJOM_MEDIA_TYPES_ENUM_MOJOM_TRAITS_H_
 
 #include "base/notreached.h"
+#include "media/base/video_frame_metadata.h"
 #include "media/base/video_transformation.h"
 #include "media/mojo/mojom/media_types.mojom-shared.h"
 
@@ -58,6 +59,41 @@ struct EnumTraits<media::mojom::VideoRotation, ::media::VideoRotation> {
   }
 };
 
+template <>
+struct EnumTraits<media::mojom::CopyMode,
+                  ::media::VideoFrameMetadata::CopyMode> {
+  static media::mojom::CopyMode ToMojom(
+      ::media::VideoFrameMetadata::CopyMode input) {
+    switch (input) {
+      case ::media::VideoFrameMetadata::CopyMode::kCopyToNewTexture:
+        return media::mojom::CopyMode::kCopyToNewTexture;
+      case ::media::VideoFrameMetadata::CopyMode::kCopyMailboxesOnly:
+        return media::mojom::CopyMode::kCopyMailboxesOnly;
+    }
+
+    NOTREACHED();
+    return static_cast<media::mojom::CopyMode>(input);
+  }
+
+  // Returning false results in deserialization failure and causes the
+  // message pipe receiving it to be disconnected.
+  static bool FromMojom(media::mojom::CopyMode input,
+                        media::VideoFrameMetadata::CopyMode* output) {
+    switch (input) {
+      case media::mojom::CopyMode::kCopyToNewTexture:
+        *output = ::media::VideoFrameMetadata::CopyMode::kCopyToNewTexture;
+        return true;
+      case media::mojom::CopyMode::kCopyMailboxesOnly:
+        *output = ::media::VideoFrameMetadata::CopyMode::kCopyMailboxesOnly;
+        return true;
+    }
+
+    NOTREACHED();
+    *output = static_cast<::media::VideoFrameMetadata::CopyMode>(input);
+    return false;
+  }
+};
+
 }  // namespace mojo
 
 #endif  // MEDIA_MOJO_MOJOM_MEDIA_TYPES_ENUM_MOJOM_TRAITS_H_

media/mojo/mojom/video_frame_metadata_mojom_traits.cc

@@ -36,7 +36,6 @@ bool StructTraits<media::mojom::VideoFrameMetadataDataView,
 
   // bool.
   output->allow_overlay = input.allow_overlay();
-  output->copy_required = input.copy_required();
   output->end_of_stream = input.end_of_stream();
   output->texture_owner = input.texture_owner();
   output->wants_promotion_hint = input.wants_promotion_hint();
@@ -64,6 +63,13 @@ bool StructTraits<media::mojom::VideoFrameMetadataDataView,
     output->rotation = rotation;
   }
 
+  if (input.has_copy_mode()) {
+    media::VideoFrameMetadata::CopyMode copy_mode;
+    if (!input.ReadCopyMode(&copy_mode))
+      return false;
+    output->copy_mode = copy_mode;
+  }
+
   READ_AND_ASSIGN_OPT(base::UnguessableToken, overlay_plane_id, OverlayPlaneId);
 
   READ_AND_ASSIGN_OPT(gfx::Rect, capture_update_rect, CaptureUpdateRect);
@@ -83,4 +89,4 @@ bool StructTraits<media::mojom::VideoFrameMetadataDataView,
   return true;
 }
 
-}  // namespace mojo
+}  // namespace mojo
\ No newline at end of file

media/mojo/mojom/video_frame_metadata_mojom_traits.h

@@ -33,10 +33,6 @@ struct StructTraits<media::mojom::VideoFrameMetadataDataView,
     return input.allow_overlay;
   }
 
-  static bool copy_required(const media::VideoFrameMetadata& input) {
-    return input.copy_required;
-  }
-
   static bool end_of_stream(const media::VideoFrameMetadata& input) {
     return input.end_of_stream;
   }
@@ -71,6 +67,11 @@ struct StructTraits<media::mojom::VideoFrameMetadataDataView,
 
   GENERATE_OPT_SERIALIZATION(int, capture_counter, 0)
 
+  GENERATE_OPT_SERIALIZATION(
+      media::VideoFrameMetadata::CopyMode,
+      copy_mode,
+      media::VideoFrameMetadata::CopyMode::kCopyToNewTexture)
+
   GENERATE_OPT_SERIALIZATION(media::VideoRotation,
                              rotation,
                              media::VideoRotation::VIDEO_ROTATION_0)

media/mojo/mojom/video_frame_metadata_mojom_traits_unittest.cc

@@ -64,7 +64,7 @@ TEST_F(VideoFrameMetadataStructTraitsTest, EmptyMetadata) {
   EXPECT_FALSE(metadata_out.capture_update_rect.has_value());
   EXPECT_FALSE(metadata_out.rotation.has_value());
   EXPECT_FALSE(metadata_out.allow_overlay);
-  EXPECT_FALSE(metadata_out.copy_required);
+  EXPECT_FALSE(metadata_out.copy_mode.has_value());
   EXPECT_FALSE(metadata_out.end_of_stream);
   EXPECT_FALSE(metadata_out.texture_owner);
   EXPECT_FALSE(metadata_out.wants_promotion_hint);
@@ -107,9 +107,12 @@ TEST_F(VideoFrameMetadataStructTraitsTest, ValidMetadata) {
   // media::VideoRotations
   metadata_in.rotation = media::VideoRotation::VIDEO_ROTATION_90;
 
+  // media::VideoFrameMetadata::CopyMode
+  metadata_in.copy_mode =
+      media::VideoFrameMetadata::CopyMode::kCopyToNewTexture;
+
   // bools
   metadata_in.allow_overlay = true;
-  metadata_in.copy_required = true;
   metadata_in.end_of_stream = true;
   metadata_in.texture_owner = true;
   metadata_in.wants_promotion_hint = true;
@@ -154,7 +157,7 @@ TEST_F(VideoFrameMetadataStructTraitsTest, ValidMetadata) {
   EXPECT_EQ(metadata_in.capture_update_rect, metadata_out.capture_update_rect);
   EXPECT_EQ(metadata_in.rotation, metadata_out.rotation);
   EXPECT_EQ(metadata_in.allow_overlay, metadata_out.allow_overlay);
-  EXPECT_EQ(metadata_in.copy_required, metadata_out.copy_required);
+  EXPECT_EQ(metadata_in.copy_mode, metadata_out.copy_mode);
   EXPECT_EQ(metadata_in.end_of_stream, metadata_out.end_of_stream);
   EXPECT_EQ(metadata_in.texture_owner, metadata_out.texture_owner);
   EXPECT_EQ(metadata_in.wants_promotion_hint,

media/renderers/video_resource_updater.cc

@@ -841,11 +841,12 @@ VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
   VideoFrameExternalResources external_resources;
   gfx::ColorSpace resource_color_space = video_frame->ColorSpace();
 
-  bool copy_required = video_frame->metadata()->copy_required;
+  bool copy_to_new_texture = (video_frame->metadata()->copy_mode ==
+                              VideoFrameMetadata::CopyMode::kCopyToNewTexture);
 
   GLuint target = video_frame->mailbox_holder(0).texture_target;
-  // If |copy_required| then we will copy into a GL_TEXTURE_2D target.
-  if (copy_required)
+  // If |copy_to_new_texture| then we will copy into a GL_TEXTURE_2D target.
+  if (copy_to_new_texture)
     target = GL_TEXTURE_2D;
 
   gfx::BufferFormat buffer_formats[VideoFrame::kMaxPlanes];
@@ -870,7 +871,7 @@ VideoFrameExternalResources VideoResourceUpdater::CreateForHardwarePlanes(
     if (mailbox_holder.mailbox.IsZero())
       break;
 
-    if (copy_required) {
+    if (copy_to_new_texture) {
       CopyHardwarePlane(video_frame.get(), resource_color_space, mailbox_holder,
                         &external_resources);
     } else {

media/renderers/video_resource_updater_unittest.cc

@@ -204,7 +204,10 @@ class VideoResourceUpdaterTest : public testing::Test {
       bool needs_copy) {
     scoped_refptr<media::VideoFrame> video_frame = CreateTestHardwareVideoFrame(
         media::PIXEL_FORMAT_ARGB, GL_TEXTURE_EXTERNAL_OES);
-    video_frame->metadata()->copy_required = needs_copy;
+    if (needs_copy) {
+      video_frame->metadata()->copy_mode =
+          VideoFrameMetadata::CopyMode::kCopyToNewTexture;
+    }
     return video_frame;
   }