media/gpu/chromeos: Add VdVideoDecodeAccelerator.

Currently GpuArcVideoDecodeAccelerator bridges the video codec from
ARC++ to a VDA instance. We plan to deprecate to VDA implementation at
chromium by new VD implementation. So we temporary need an adapter to
convert between these two interfaces, until the
GpuArcVideoDecodeAccelerator is ported to the new VD interface.

This CL implements VdVideoDecodeAccelerator as the adapter. With this
class, GpuArcVideoDecodeAccelerator could bridge the video codec to a
VD instance via the adapter.

Bug: b:136716838
Test: Pass video_decode_accelerator_tests with following CLs
      on Kevin and eve

Change-Id: I17f8607cc422c682eaad7760f5680a425de90095
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1913228
Commit-Queue: Chih-Yu Huang <akahuang@chromium.org>
Reviewed-by: Alexandre Courbot <acourbot@chromium.org>
Reviewed-by: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: David Staessens <dstaessens@chromium.org>
Cr-Commit-Position: refs/heads/master@{#748086}

media/gpu/chromeos/BUILD.gn

@@ -62,6 +62,10 @@ source_set("common") {
     "platform_video_frame_pool.h",
     "platform_video_frame_utils.cc",
     "platform_video_frame_utils.h",
+    "vd_video_decode_accelerator.cc",
+    "vd_video_decode_accelerator.h",
+    "vda_video_frame_pool.cc",
+    "vda_video_frame_pool.h",
     "video_decoder_pipeline.cc",
     "video_decoder_pipeline.h",
     "video_frame_converter.cc",

media/gpu/chromeos/vd_video_decode_accelerator.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/gpu/chromeos/vd_video_decode_accelerator.h"
+
+#include <memory>
+#include <vector>
+
+#include "base/bind.h"
+#include "base/location.h"
+#include "base/macros.h"
+#include "media/base/video_color_space.h"
+#include "media/base/video_decoder_config.h"
+#include "media/base/video_frame.h"
+#include "media/base/video_frame_metadata.h"
+#include "media/base/video_transformation.h"
+#include "media/base/video_types.h"
+#include "media/base/waiting.h"
+#include "media/gpu/chromeos/gpu_buffer_layout.h"
+#include "media/gpu/macros.h"
+#include "ui/gl/gl_bindings.h"
+
+namespace media {
+namespace {
+
+// VideoDecoder copies the timestamp from DecodeBuffer to its corresponding
+// VideoFrame. However, VideoDecodeAccelerator uses bitstream ID to find the
+// corresponding output picture. Therefore, we store bitstream ID at the
+// timestamp field. These two functions are used for converting between
+// bitstream ID and fake timestamp.
+base::TimeDelta BitstreamIdToFakeTimestamp(int32_t bitstream_id) {
+  return base::TimeDelta::FromMilliseconds(bitstream_id);
+}
+
+int32_t FakeTimestampToBitstreamId(base::TimeDelta timestamp) {
+  return static_cast<int32_t>(timestamp.InMilliseconds());
+}
+
+std::vector<ColorPlaneLayout> ExtractColorPlaneLayout(
+    const gfx::GpuMemoryBufferHandle& gmb_handle) {
+  std::vector<ColorPlaneLayout> planes;
+  for (const auto& plane : gmb_handle.native_pixmap_handle.planes)
+    planes.emplace_back(plane.stride, plane.offset, plane.size);
+  return planes;
+}
+
+std::vector<base::ScopedFD> ExtractFds(gfx::GpuMemoryBufferHandle gmb_handle) {
+  std::vector<base::ScopedFD> fds;
+  for (auto& plane : gmb_handle.native_pixmap_handle.planes)
+    fds.push_back(std::move(plane.fd));
+  return fds;
+}
+
+// TODO(akahuang): Move this function to a utility file.
+template <class T>
+std::string VectorToString(const std::vector<T>& vec) {
+  std::ostringstream result;
+  std::string delim;
+  result << "[";
+  for (auto& v : vec) {
+    result << delim << v;
+    if (delim.size() == 0)
+      delim = ", ";
+  }
+  result << "]";
+  return result.str();
+}
+
+}  // namespace
+
+// static
+std::unique_ptr<VideoDecodeAccelerator> VdVideoDecodeAccelerator::Create(
+    CreateVideoDecoderCb create_vd_cb,
+    Client* client,
+    const Config& config,
+    scoped_refptr<base::SequencedTaskRunner> task_runner) {
+  std::unique_ptr<VideoDecodeAccelerator> vda(new VdVideoDecodeAccelerator(
+      std::move(create_vd_cb), std::move(task_runner)));
+  if (!vda->Initialize(config, client))
+    return nullptr;
+  return vda;
+}
+
+VdVideoDecodeAccelerator::VdVideoDecodeAccelerator(
+    CreateVideoDecoderCb create_vd_cb,
+    scoped_refptr<base::SequencedTaskRunner> client_task_runner)
+    : create_vd_cb_(std::move(create_vd_cb)),
+      client_task_runner_(std::move(client_task_runner)) {
+  VLOGF(2);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  weak_this_ = weak_this_factory_.GetWeakPtr();
+}
+
+void VdVideoDecodeAccelerator::Destroy() {
+  VLOGF(2);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  weak_this_factory_.InvalidateWeakPtrs();
+
+  // Because VdaVideoFramePool is blocked for this callback, we must call the
+  // callback before destroying.
+  if (notify_layout_changed_cb_)
+    std::move(notify_layout_changed_cb_).Run(base::nullopt);
+  client_ = nullptr;
+  vd_.reset();
+
+  delete this;
+}
+
+VdVideoDecodeAccelerator::~VdVideoDecodeAccelerator() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+}
+
+bool VdVideoDecodeAccelerator::Initialize(const Config& config,
+                                          Client* client) {
+  VLOGF(2) << "config: " << config.AsHumanReadableString();
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(!client_);
+
+  if (config.is_encrypted()) {
+    VLOGF(1) << "Encrypted streams are not supported";
+    return false;
+  }
+  if (config.output_mode != Config::OutputMode::IMPORT) {
+    VLOGF(1) << "Only IMPORT OutputMode is supported.";
+    return false;
+  }
+  if (!config.is_deferred_initialization_allowed) {
+    VLOGF(1) << "Only is_deferred_initialization_allowed is supported.";
+    return false;
+  }
+
+  std::unique_ptr<VdaVideoFramePool> frame_pool =
+      std::make_unique<VdaVideoFramePool>(weak_this_, client_task_runner_);
+  vd_ = create_vd_cb_.Run(client_task_runner_, std::move(frame_pool),
+                          std::make_unique<VideoFrameConverter>(),
+                          nullptr /* gpu_memory_buffer_factory */);
+  if (!vd_)
+    return false;
+
+  client_ = client;
+
+  VideoDecoderConfig vd_config(
+      VideoCodecProfileToVideoCodec(config.profile), config.profile,
+      VideoDecoderConfig::AlphaMode::kIsOpaque, config.container_color_space,
+      VideoTransformation(), config.initial_expected_coded_size,
+      gfx::Rect(config.initial_expected_coded_size),
+      config.initial_expected_coded_size, std::vector<uint8_t>(),
+      EncryptionScheme::kUnencrypted);
+  auto init_cb =
+      base::BindOnce(&VdVideoDecodeAccelerator::OnInitializeDone, weak_this_);
+  auto output_cb =
+      base::BindRepeating(&VdVideoDecodeAccelerator::OnFrameReady, weak_this_);
+  vd_->Initialize(std::move(vd_config), false /* low_delay */,
+                  nullptr /* cdm_context */, std::move(init_cb),
+                  std::move(output_cb), WaitingCB());
+  return true;
+}
+
+void VdVideoDecodeAccelerator::OnInitializeDone(bool success) {
+  DVLOGF(3) << "success: " << success;
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(client_);
+
+  client_->NotifyInitializationComplete(success);
+}
+
+void VdVideoDecodeAccelerator::Decode(BitstreamBuffer bitstream_buffer) {
+  Decode(bitstream_buffer.ToDecoderBuffer(), bitstream_buffer.id());
+}
+
+void VdVideoDecodeAccelerator::Decode(scoped_refptr<DecoderBuffer> buffer,
+                                      int32_t bitstream_id) {
+  DVLOGF(4) << "bitstream_id:" << bitstream_id;
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(vd_);
+
+  // Set timestamp field as bitstream buffer id, because we can only use
+  // timestamp field to find the corresponding output frames. Also, VDA doesn't
+  // care about timestamp.
+  buffer->set_timestamp(BitstreamIdToFakeTimestamp(bitstream_id));
+
+  vd_->Decode(std::move(buffer),
+              base::BindOnce(&VdVideoDecodeAccelerator::OnDecodeDone,
+                             weak_this_, bitstream_id));
+}
+
+void VdVideoDecodeAccelerator::OnDecodeDone(int32_t bitstream_buffer_id,
+                                            DecodeStatus status) {
+  DVLOGF(4) << "status: " << status;
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(client_);
+
+  if (status == DecodeStatus::DECODE_ERROR) {
+    OnError(FROM_HERE, PLATFORM_FAILURE);
+    return;
+  }
+
+  client_->NotifyEndOfBitstreamBuffer(bitstream_buffer_id);
+}
+
+void VdVideoDecodeAccelerator::OnFrameReady(scoped_refptr<VideoFrame> frame) {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(frame);
+  DCHECK(client_);
+
+  base::Optional<Picture> picture = GetPicture(*frame);
+  if (!picture) {
+    VLOGF(1) << "Failed to get picture.";
+    OnError(FROM_HERE, PLATFORM_FAILURE);
+    return;
+  }
+
+  // Record that the picture is sent to the client.
+  auto it = picture_at_client_.find(picture->picture_buffer_id());
+  if (it == picture_at_client_.end()) {
+    // We haven't sent the buffer to the client. Set |num_sent| = 1;
+    picture_at_client_.emplace(picture->picture_buffer_id(),
+                               std::make_pair(std::move(frame), 1));
+  } else {
+    // We already sent the buffer to the client (only happen when using VP9
+    // show_existing_frame feature). Increase |num_sent|;
+    ++(it->second.second);
+  }
+
+  client_->PictureReady(*picture);
+}
+
+void VdVideoDecodeAccelerator::Flush() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(vd_);
+
+  vd_->Decode(
+      DecoderBuffer::CreateEOSBuffer(),
+      base::BindOnce(&VdVideoDecodeAccelerator::OnFlushDone, weak_this_));
+}
+
+void VdVideoDecodeAccelerator::OnFlushDone(DecodeStatus status) {
+  DVLOGF(3) << "status: " << status;
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(client_);
+
+  switch (status) {
+    case DecodeStatus::OK:
+      client_->NotifyFlushDone();
+      break;
+    case DecodeStatus::ABORTED:
+      // Do nothing.
+      break;
+    case DecodeStatus::DECODE_ERROR:
+      OnError(FROM_HERE, PLATFORM_FAILURE);
+      break;
+  }
+}
+
+void VdVideoDecodeAccelerator::Reset() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(vd_);
+
+  vd_->Reset(
+      base::BindOnce(&VdVideoDecodeAccelerator::OnResetDone, weak_this_));
+}
+
+void VdVideoDecodeAccelerator::OnResetDone() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(client_);
+
+  client_->NotifyResetDone();
+}
+
+void VdVideoDecodeAccelerator::RequestFrames(
+    const Fourcc& fourcc,
+    const gfx::Size& coded_size,
+    const gfx::Rect& visible_rect,
+    size_t max_num_frames,
+    NotifyLayoutChangedCb notify_layout_changed_cb,
+    ImportFrameCb import_frame_cb) {
+  VLOGF(2);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+  DCHECK(client_);
+  DCHECK(!notify_layout_changed_cb_);
+
+  notify_layout_changed_cb_ = std::move(notify_layout_changed_cb);
+  import_frame_cb_ = std::move(import_frame_cb);
+
+  // After calling ProvidePictureBuffersWithVisibleRect(), the client might
+  // still send buffers with old coded size. We temporarily store at
+  // |pending_coded_size_|.
+  pending_coded_size_ = coded_size;
+  client_->ProvidePictureBuffersWithVisibleRect(
+      max_num_frames, fourcc.ToVideoPixelFormat(), 1 /* textures_per_buffer */,
+      coded_size, visible_rect, GL_TEXTURE_EXTERNAL_OES);
+}
+
+void VdVideoDecodeAccelerator::AssignPictureBuffers(
+    const std::vector<PictureBuffer>& buffers) {
+  VLOGF(2);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  // After AssignPictureBuffers() is called, the buffers sent from
+  // ImportBufferForPicture() should be with new coded size. Now we can update
+  // |coded_size_|.
+  coded_size_ = pending_coded_size_;
+}
+
+void VdVideoDecodeAccelerator::ImportBufferForPicture(
+    int32_t picture_buffer_id,
+    VideoPixelFormat pixel_format,
+    gfx::GpuMemoryBufferHandle gmb_handle) {
+  DVLOGF(4) << "picture_buffer_id: " << picture_buffer_id;
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  // The first imported picture after requesting buffers.
+  // |notify_layout_changed_cb_| must be called in this clause because it blocks
+  // VdaVideoFramePool.
+  if (notify_layout_changed_cb_) {
+    auto fourcc = Fourcc::FromVideoPixelFormat(pixel_format);
+    if (!fourcc) {
+      VLOGF(1) << "Failed to convert to Fourcc.";
+      std::move(notify_layout_changed_cb_).Run(base::nullopt);
+      return;
+    }
+
+    std::vector<ColorPlaneLayout> planes = ExtractColorPlaneLayout(gmb_handle);
+    layout_ =
+        VideoFrameLayout::CreateWithPlanes(pixel_format, coded_size_, planes);
+    if (!layout_) {
+      VLOGF(1) << "Failed to create VideoFrameLayout. format: "
+               << VideoPixelFormatToString(pixel_format)
+               << ", coded_size: " << coded_size_.ToString()
+               << ", planes: " << VectorToString(planes);
+      std::move(notify_layout_changed_cb_).Run(base::nullopt);
+      return;
+    }
+
+    std::move(notify_layout_changed_cb_)
+        .Run(GpuBufferLayout::Create(*fourcc, coded_size_, planes));
+  }
+
+  if (!layout_)
+    return;
+
+  // VideoFrame::WrapVideoFrame() will check whether the updated visible_rect
+  // is sub rect of the original visible_rect. Therefore we set visible_rect
+  // as large as coded_size to guarantee this condition.
+  scoped_refptr<VideoFrame> origin_frame = VideoFrame::WrapExternalDmabufs(
+      *layout_, gfx::Rect(coded_size_), coded_size_,
+      ExtractFds(std::move(gmb_handle)), base::TimeDelta());
+  DmabufId dmabuf_id = DmabufVideoFramePool::GetDmabufId(*origin_frame);
+  auto res = frame_id_to_picture_id_.emplace(dmabuf_id, picture_buffer_id);
+  // |dmabuf_id| should not be inside the map before insertion.
+  DCHECK(res.second);
+
+  // |wrapped_frame| is used to keep |origin_frame| alive until everyone
+  // released |wrapped_frame|. Then DmabufId will be available at
+  // OnFrameReleased().
+  scoped_refptr<VideoFrame> wrapped_frame = VideoFrame::WrapVideoFrame(
+      origin_frame, origin_frame->format(), origin_frame->visible_rect(),
+      origin_frame->natural_size());
+  wrapped_frame->AddDestructionObserver(
+      base::BindOnce(&VdVideoDecodeAccelerator::OnFrameReleasedThunk,
+                     weak_this_, client_task_runner_, std::move(origin_frame)));
+
+  DCHECK(import_frame_cb_);
+  import_frame_cb_.Run(std::move(wrapped_frame));
+}
+
+base::Optional<Picture> VdVideoDecodeAccelerator::GetPicture(
+    const VideoFrame& frame) {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  auto it =
+      frame_id_to_picture_id_.find(DmabufVideoFramePool::GetDmabufId(frame));
+  if (it == frame_id_to_picture_id_.end()) {
+    VLOGF(1) << "Failed to find the picture buffer id.";
+    return base::nullopt;
+  }
+  int32_t picture_buffer_id = it->second;
+  int32_t bitstream_id = FakeTimestampToBitstreamId(frame.timestamp());
+  bool allow_overlay = false;
+  ignore_result(frame.metadata()->GetBoolean(VideoFrameMetadata::ALLOW_OVERLAY,
+                                             &allow_overlay));
+  return base::make_optional(Picture(picture_buffer_id, bitstream_id,
+                                     frame.visible_rect(), frame.ColorSpace(),
+                                     allow_overlay));
+}
+
+// static
+void VdVideoDecodeAccelerator::OnFrameReleasedThunk(
+    base::Optional<base::WeakPtr<VdVideoDecodeAccelerator>> weak_this,
+    scoped_refptr<base::SequencedTaskRunner> task_runner,
+    scoped_refptr<VideoFrame> origin_frame) {
+  DVLOGF(4);
+  DCHECK(weak_this);
+
+  task_runner->PostTask(
+      FROM_HERE, base::BindOnce(&VdVideoDecodeAccelerator::OnFrameReleased,
+                                *weak_this, std::move(origin_frame)));
+}
+
+void VdVideoDecodeAccelerator::OnFrameReleased(
+    scoped_refptr<VideoFrame> origin_frame) {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  auto it = frame_id_to_picture_id_.find(
+      DmabufVideoFramePool::GetDmabufId(*origin_frame));
+  DCHECK(it != frame_id_to_picture_id_.end());
+  int32_t picture_buffer_id = it->second;
+  frame_id_to_picture_id_.erase(it);
+
+  client_->DismissPictureBuffer(picture_buffer_id);
+}
+
+void VdVideoDecodeAccelerator::ReusePictureBuffer(int32_t picture_buffer_id) {
+  DVLOGF(4) << "picture_buffer_id: " << picture_buffer_id;
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  auto it = picture_at_client_.find(picture_buffer_id);
+  if (it == picture_at_client_.end()) {
+    DVLOGF(3) << picture_buffer_id << " has already been dismissed, ignore.";
+    return;
+  }
+
+  size_t& num_sent = it->second.second;
+  DCHECK_NE(num_sent, 0u);
+  --num_sent;
+
+  // The count of calling VDA::ReusePictureBuffer() is the same as calling
+  // Client::PictureReady(). Now we could really reuse the buffer.
+  if (num_sent == 0)
+    picture_at_client_.erase(it);
+}
+
+void VdVideoDecodeAccelerator::OnError(base::Location location, Error error) {
+  LOG(ERROR) << "Failed at " << location.ToString()
+             << ", error code: " << static_cast<int>(error);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(client_sequence_checker_);
+
+  client_->NotifyError(error);
+}
+
+}  // namespace media

media/gpu/chromeos/vd_video_decode_accelerator.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_GPU_CHROMEOS_VD_VIDEO_DECODE_ACCELERATOR_H_
+#define MEDIA_GPU_CHROMEOS_VD_VIDEO_DECODE_ACCELERATOR_H_
+
+#include <map>
+#include <memory>
+
+#include "base/callback_forward.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/sequence_checker.h"
+#include "base/sequenced_task_runner.h"
+#include "base/time/time.h"
+#include "media/base/video_decoder.h"
+#include "media/gpu/chromeos/dmabuf_video_frame_pool.h"
+#include "media/gpu/chromeos/fourcc.h"
+#include "media/gpu/chromeos/vda_video_frame_pool.h"
+#include "media/gpu/chromeos/video_frame_converter.h"
+#include "media/gpu/media_gpu_export.h"
+#include "media/video/video_decode_accelerator.h"
+
+namespace gpu {
+class GpuMemoryBufferFactory;
+}  // namespace gpu
+
+namespace media {
+
+class VideoFrame;
+
+// Implements the VideoDecodeAccelerator backed by a VideoDecoder.
+// Currently GpuArcVideoDecodeAccelerator bridges the video codec from
+// ARC++ to a VDA instance. We plan to deprecate to VDA implementation at
+// chromium by new VD implementation. So we need the adapter between
+// these two interface.
+//
+// Important note: This adaptor is only used temporary as an intermediate step
+// while the GpuArcVideoDecodeAccelerator is being ported to the new VD
+// interface. This Adaptor will be deprecated soon and should not be used
+// anywhere else.
+class MEDIA_GPU_EXPORT VdVideoDecodeAccelerator
+    : public VideoDecodeAccelerator,
+      public VdaVideoFramePool::VdaDelegate {
+ public:
+  // Callback for creating VideoDecoder instance.
+  using CreateVideoDecoderCb =
+      base::RepeatingCallback<std::unique_ptr<VideoDecoder>(
+          scoped_refptr<base::SequencedTaskRunner>,
+          std::unique_ptr<DmabufVideoFramePool>,
+          std::unique_ptr<VideoFrameConverter>,
+          gpu::GpuMemoryBufferFactory* const)>;
+
+  // Create VdVideoDecodeAccelerator instance, and call Initialize().
+  // Return nullptr if Initialize() failed.
+  static std::unique_ptr<VideoDecodeAccelerator> Create(
+      CreateVideoDecoderCb create_vd_cb,
+      Client* client,
+      const Config& config,
+      scoped_refptr<base::SequencedTaskRunner> task_runner);
+  VdVideoDecodeAccelerator(const VdVideoDecodeAccelerator&) = delete;
+  VdVideoDecodeAccelerator& operator=(const VdVideoDecodeAccelerator&) = delete;
+  ~VdVideoDecodeAccelerator() override;
+
+  // Implementation of VideoDecodeAccelerator.
+  bool Initialize(const Config& config, Client* client) override;
+  void AssignPictureBuffers(const std::vector<PictureBuffer>& buffers) override;
+  void ImportBufferForPicture(
+      int32_t picture_buffer_id,
+      VideoPixelFormat pixel_format,
+      gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle) override;
+  void ReusePictureBuffer(int32_t picture_buffer_id) override;
+  void Decode(BitstreamBuffer bitstream_buffer) override;
+  void Decode(scoped_refptr<DecoderBuffer> buffer,
+              int32_t bitstream_id) override;
+  void Flush() override;
+  void Reset() override;
+  void Destroy() override;
+
+  // Implementation of VdaVideoFramePool::VdaDelegate.
+  void RequestFrames(const Fourcc& fourcc,
+                     const gfx::Size& coded_size,
+                     const gfx::Rect& visible_rect,
+                     size_t max_num_frames,
+                     NotifyLayoutChangedCb notify_layout_changed_cb,
+                     ImportFrameCb import_frame_cb) override;
+
+ private:
+  using DmabufId = DmabufVideoFramePool::DmabufId;
+
+  VdVideoDecodeAccelerator(
+      CreateVideoDecoderCb create_vd_cb,
+      scoped_refptr<base::SequencedTaskRunner> task_runner);
+
+  // Callback methods of |vd_|.
+  void OnInitializeDone(bool success);
+  void OnDecodeDone(int32_t bitstream_buffer_id, DecodeStatus status);
+  void OnFrameReady(scoped_refptr<VideoFrame> frame);
+  void OnFlushDone(DecodeStatus status);
+  void OnResetDone();
+
+  // Get Picture instance that represents the same buffer as |frame|. Return
+  // base::nullopt if the buffer is already dismissed.
+  base::Optional<Picture> GetPicture(const VideoFrame& frame);
+
+  // Thunk to post OnFrameReleased() to |task_runner|.
+  // Because this thunk may be called in any thread, We don't want to
+  // dereference WeakPtr. Therefore we wrap the WeakPtr by base::Optional to
+  // avoid the task runner defererencing the WeakPtr.
+  static void OnFrameReleasedThunk(
+      base::Optional<base::WeakPtr<VdVideoDecodeAccelerator>> weak_this,
+      scoped_refptr<base::SequencedTaskRunner> task_runner,
+      scoped_refptr<VideoFrame> origin_frame);
+  // Called when a frame gets destroyed.
+  void OnFrameReleased(scoped_refptr<VideoFrame> origin_frame);
+
+  // Called when any error occurs. Notify |client_| an error occurred.
+  void OnError(base::Location location, Error error);
+
+  // Callback to generate VideoDecoder.
+  CreateVideoDecoderCb create_vd_cb_;
+  // The client of this VDA.
+  VideoDecodeAccelerator::Client* client_ = nullptr;
+  // The delegated VideoDecoder instance.
+  std::unique_ptr<VideoDecoder> vd_;
+  // Callback for returning the result after this instance is asked to request
+  // new frames. The VdaVideoFramePool is blocked until this callback is called.
+  NotifyLayoutChangedCb notify_layout_changed_cb_;
+  // Callback for passing the available frames to the pool.
+  ImportFrameCb import_frame_cb_;
+
+  // The size requested from VdaVideoFramePool.
+  gfx::Size pending_coded_size_;
+  // The formats of the current buffers.
+  gfx::Size coded_size_;
+  base::Optional<VideoFrameLayout> layout_;
+
+  // Mapping from VideoFrame's DmabufId to picture buffer id.
+  std::map<DmabufId, int32_t /* picture_buffer_id */> frame_id_to_picture_id_;
+  // Record how many times the picture is sent to the client, and keep a refptr
+  // of corresponding VideoFrame when the client owns the buffers.
+  std::map<int32_t /* picture_buffer_id */,
+           std::pair<scoped_refptr<VideoFrame>, size_t /* num_sent */>>
+      picture_at_client_;
+
+  // Main task runner and its sequence checker. All methods should be called
+  // on it.
+  scoped_refptr<base::SequencedTaskRunner> client_task_runner_;
+  SEQUENCE_CHECKER(client_sequence_checker_);
+
+  // The weak pointer of this class instance, bound to |client_task_runner_|.
+  base::WeakPtr<VdVideoDecodeAccelerator> weak_this_;
+  base::WeakPtrFactory<VdVideoDecodeAccelerator> weak_this_factory_{this};
+};
+
+}  // namespace media
+
+#endif  // MEDIA_GPU_CHROMEOS_VD_VIDEO_DECODE_ACCELERATOR_H_

media/gpu/chromeos/vda_video_frame_pool.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/gpu/chromeos/vda_video_frame_pool.h"
+
+#include "base/bind.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/synchronization/waitable_event.h"
+#include "media/gpu/chromeos/gpu_buffer_layout.h"
+#include "media/gpu/macros.h"
+
+namespace media {
+
+VdaVideoFramePool::VdaVideoFramePool(
+    base::WeakPtr<VdaDelegate> vda,
+    scoped_refptr<base::SequencedTaskRunner> vda_task_runner)
+    : vda_(std::move(vda)), vda_task_runner_(std::move(vda_task_runner)) {
+  DVLOGF(3);
+  DETACH_FROM_SEQUENCE(parent_sequence_checker_);
+
+  weak_this_ = weak_this_factory_.GetWeakPtr();
+}
+
+VdaVideoFramePool::~VdaVideoFramePool() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  weak_this_factory_.InvalidateWeakPtrs();
+}
+
+base::Optional<GpuBufferLayout> VdaVideoFramePool::Initialize(
+    const Fourcc& fourcc,
+    const gfx::Size& coded_size,
+    const gfx::Rect& visible_rect,
+    const gfx::Size& natural_size,
+    size_t max_num_frames) {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  visible_rect_ = visible_rect;
+  natural_size_ = natural_size;
+
+  if (max_num_frames_ == max_num_frames && fourcc_ && *fourcc_ == fourcc &&
+      coded_size_ == coded_size) {
+    DVLOGF(3) << "Arguments related to frame layout are not changed, skip.";
+    return layout_;
+  }
+  max_num_frames_ = max_num_frames;
+  fourcc_ = fourcc;
+  coded_size_ = coded_size;
+
+  // Clear the pool and reset the layout to prevent previous frames are recycled
+  // back to the pool.
+  frame_pool_ = {};
+  layout_ = base::nullopt;
+
+  // Receive the layout from the callback. |layout_| is accessed on
+  // |parent_task_runner_| except OnRequestFramesDone(). However, we block
+  // |parent_task_runner_| until OnRequestFramesDone() returns. So we don't need
+  // a lock to protect |layout_|.
+  // Also it's safe to use base::Unretained() here because we block here, |this|
+  // must be alive during the callback.
+  base::WaitableEvent done;
+  vda_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(&VdaDelegate::RequestFrames, vda_, fourcc, coded_size,
+                     visible_rect, max_num_frames,
+                     base::BindOnce(&VdaVideoFramePool::OnRequestFramesDone,
+                                    base::Unretained(this), &done),
+                     base::BindRepeating(&VdaVideoFramePool::ImportFrameThunk,
+                                         parent_task_runner_, weak_this_)));
+  done.Wait();
+
+  return layout_;
+}
+
+void VdaVideoFramePool::OnRequestFramesDone(
+    base::WaitableEvent* done,
+    base::Optional<GpuBufferLayout> value) {
+  DVLOGF(3);
+  // RequestFrames() is blocked on |parent_task_runner_| to wait for this method
+  // finishes, so this method must not be run on the same sequence.
+  DCHECK(!parent_task_runner_->RunsTasksInCurrentSequence());
+
+  DCHECK(fourcc_);
+  DCHECK_EQ(value->fourcc(), *fourcc_);
+  DCHECK_GE(value->size().height(), coded_size_.height());
+  DCHECK_GE(value->size().width(), coded_size_.width());
+
+  layout_ = value;
+  done->Signal();
+}
+
+// static
+void VdaVideoFramePool::ImportFrameThunk(
+    scoped_refptr<base::SequencedTaskRunner> task_runner,
+    base::Optional<base::WeakPtr<VdaVideoFramePool>> weak_this,
+    scoped_refptr<VideoFrame> frame) {
+  DVLOGF(3);
+  DCHECK(weak_this);
+
+  task_runner->PostTask(
+      FROM_HERE, base::BindOnce(&VdaVideoFramePool::ImportFrame, *weak_this,
+                                std::move(frame)));
+}
+
+void VdaVideoFramePool::ImportFrame(scoped_refptr<VideoFrame> frame) {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  if (!layout_ || layout_->fourcc().ToVideoPixelFormat() != frame->format() ||
+      layout_->size() != frame->coded_size()) {
+    return;
+  }
+
+  frame_pool_.push(std::move(frame));
+  CallFrameAvailableCbIfNeeded();
+}
+
+scoped_refptr<VideoFrame> VdaVideoFramePool::GetFrame() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  if (IsExhausted())
+    return nullptr;
+
+  scoped_refptr<VideoFrame> origin_frame = std::move(frame_pool_.front());
+  frame_pool_.pop();
+
+  // Update visible_rect and natural_size.
+  scoped_refptr<VideoFrame> wrapped_frame = VideoFrame::WrapVideoFrame(
+      origin_frame, origin_frame->format(), visible_rect_, natural_size_);
+  DCHECK(wrapped_frame);
+  wrapped_frame->AddDestructionObserver(
+      base::BindOnce(&VdaVideoFramePool::ImportFrameThunk, parent_task_runner_,
+                     weak_this_, std::move(origin_frame)));
+  return wrapped_frame;
+}
+
+bool VdaVideoFramePool::IsExhausted() {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  return frame_pool_.empty();
+}
+
+void VdaVideoFramePool::NotifyWhenFrameAvailable(base::OnceClosure cb) {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  frame_available_cb_ = std::move(cb);
+  CallFrameAvailableCbIfNeeded();
+}
+
+void VdaVideoFramePool::CallFrameAvailableCbIfNeeded() {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  if (frame_available_cb_ && !IsExhausted())
+    parent_task_runner_->PostTask(FROM_HERE, std::move(frame_available_cb_));
+}
+
+}  // namespace media

media/gpu/chromeos/vda_video_frame_pool.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_GPU_CHROMEOS_VDA_VIDEO_FRAME_POOL_H_
+#define MEDIA_GPU_CHROMEOS_VDA_VIDEO_FRAME_POOL_H_
+
+#include <vector>
+
+#include "base/containers/queue.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/sequenced_task_runner.h"
+#include "media/base/video_frame.h"
+#include "media/gpu/chromeos/dmabuf_video_frame_pool.h"
+#include "media/gpu/chromeos/fourcc.h"
+
+namespace base {
+class WaitableEvent;
+}
+
+namespace media {
+
+class GpuBufferLayout;
+
+// This class is used by VdVideoDecodeAccelerator, which adapts
+// VideoDecodeAccelerator to VideoDecoder interface.
+// The mission is to allocate and manage DMA-buf VideoFrame by delegating the
+// requests of buffer allocation to a VideoDecodeAccelerator instance, and
+// provide VideoFrame to the VideoDecoder instance.
+// The communication with VdVideoDecodeAccelerator, which inherits
+// VdaDelegate, is executed on |vda_task_runner_|, and the communication with
+// VideoDecoder instance is on |parent_task_runner_|.
+class VdaVideoFramePool : public DmabufVideoFramePool {
+ public:
+  class VdaDelegate {
+   public:
+    // Callback for returning the layout of requested buffer.
+    using NotifyLayoutChangedCb =
+        base::OnceCallback<void(base::Optional<GpuBufferLayout>)>;
+    // Callback for importing available frames to this pool.
+    using ImportFrameCb =
+        base::RepeatingCallback<void(scoped_refptr<VideoFrame>)>;
+
+    // Request new frames from VDA's client. VdaDelegate has to return the
+    // layout of frames by calling |notify_layout_changed_cb|.
+    // After that, VdaDelegate should pass frames by calling
+    // |import_frame_cb|.
+    // Note: RequestFrames(), |notify_layout_changed_cb|, and |import_frame_cb|
+    // should be called on VdaVideoFramePool's |vda_task_runner_|.
+    virtual void RequestFrames(const Fourcc& fourcc,
+                               const gfx::Size& coded_size,
+                               const gfx::Rect& visible_rect,
+                               size_t max_num_frames,
+                               NotifyLayoutChangedCb notify_layout_changed_cb,
+                               ImportFrameCb import_frame_cb) = 0;
+  };
+
+  VdaVideoFramePool(base::WeakPtr<VdaDelegate> vda,
+                    scoped_refptr<base::SequencedTaskRunner> vda_task_runner);
+  ~VdaVideoFramePool() override;
+
+  // DmabufVideoFramePool implementation.
+  base::Optional<GpuBufferLayout> Initialize(const Fourcc& fourcc,
+                                             const gfx::Size& coded_size,
+                                             const gfx::Rect& visible_rect,
+                                             const gfx::Size& natural_size,
+                                             size_t max_num_frames) override;
+  scoped_refptr<VideoFrame> GetFrame() override;
+  bool IsExhausted() override;
+  void NotifyWhenFrameAvailable(base::OnceClosure cb) override;
+
+ private:
+  // Update the layout of the buffers. |vda_| calls this as
+  // NotifyLayoutChangedCb.
+  void OnRequestFramesDone(base::WaitableEvent* done,
+                           base::Optional<GpuBufferLayout> value);
+
+  // Thunk to post ImportFrame() to |task_runner|.
+  // Because this thunk may be called in any thread, We don't want to
+  // dereference WeakPtr. Therefore we wrap the WeakPtr by base::Optional to
+  // avoid the task runner defererencing the WeakPtr.
+  static void ImportFrameThunk(
+      scoped_refptr<base::SequencedTaskRunner> task_runner,
+      base::Optional<base::WeakPtr<VdaVideoFramePool>> weak_this,
+      scoped_refptr<VideoFrame> frame);
+  // Import an available frame.
+  void ImportFrame(scoped_refptr<VideoFrame> frame);
+
+  // Call |frame_available_cb_| when the pool is not exhausted.
+  void CallFrameAvailableCbIfNeeded();
+
+  // WeakPtr of VdaDelegate instance, bound at |vda_task_runner_|.
+  base::WeakPtr<VdaDelegate> vda_;
+  // Task runner that interacts with VdaDelegate. All VdaDelegate's methods
+  // and their callbacks should be called on this task runner.
+  // Note: DmabufVideoFrame's public methods like Initialize() and GetFrame()
+  // should be called on |parent_task_runner_|.
+  scoped_refptr<base::SequencedTaskRunner> vda_task_runner_;
+
+  // The callback which is called when the pool is not exhausted.
+  base::OnceClosure frame_available_cb_;
+
+  // The layout of the frames in |frame_pool_|.
+  base::Optional<GpuBufferLayout> layout_;
+
+  // Data passed from Initialize().
+  size_t max_num_frames_ = 0;
+  base::Optional<Fourcc> fourcc_;
+  gfx::Size coded_size_;
+  gfx::Rect visible_rect_;
+  gfx::Size natural_size_;
+
+  base::queue<scoped_refptr<VideoFrame>> frame_pool_;
+
+  // Sequence checker for |vda_task_runner_|.
+  SEQUENCE_CHECKER(vda_sequence_checker_);
+  // Sequence checker for |parent_task_runner_|.
+  SEQUENCE_CHECKER(parent_sequence_checker_);
+
+  // The weak pointer of this, bound at |parent_task_runner_|.
+  base::WeakPtr<VdaVideoFramePool> weak_this_;
+  base::WeakPtrFactory<VdaVideoFramePool> weak_this_factory_{this};
+};
+
+}  // namespace media
+
+#endif  // MEDIA_GPU_CHROMEOS_VDA_VIDEO_FRAME_POOL_H_