media/gpu/chromeos: Add VdVideoDecodeAccelerator.

Currently GpuArcVideoDecodeAccelerator bridges the video codec from
ARC++ to a VDA instance. We plan to deprecate to VDA implementation at
chromium by new VD implementation. So we temporary need an adapter to
convert between these two interfaces, until the
GpuArcVideoDecodeAccelerator is ported to the new VD interface.

This CL implements VdVideoDecodeAccelerator as the adapter. With this
class, GpuArcVideoDecodeAccelerator could bridge the video codec to a
VD instance via the adapter.

Bug: b:136716838
Test: Pass video_decode_accelerator_tests with following CLs
      on Kevin and eve

Change-Id: I17f8607cc422c682eaad7760f5680a425de90095
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1913228
Commit-Queue: Chih-Yu Huang <akahuang@chromium.org>
Reviewed-by: Alexandre Courbot <acourbot@chromium.org>
Reviewed-by: Hirokazu Honda <hiroh@chromium.org>
Reviewed-by: David Staessens <dstaessens@chromium.org>
Cr-Commit-Position: refs/heads/master@{#748086}

media/gpu/chromeos/BUILD.gn

@@ -62,6 +62,10 @@ source_set("common") {
     "platform_video_frame_pool.h",
     "platform_video_frame_utils.cc",
     "platform_video_frame_utils.h",
+    "vd_video_decode_accelerator.cc",
+    "vd_video_decode_accelerator.h",
+    "vda_video_frame_pool.cc",
+    "vda_video_frame_pool.h",
     "video_decoder_pipeline.cc",
     "video_decoder_pipeline.h",
     "video_frame_converter.cc",

media/gpu/chromeos/vd_video_decode_accelerator.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_GPU_CHROMEOS_VD_VIDEO_DECODE_ACCELERATOR_H_
+#define MEDIA_GPU_CHROMEOS_VD_VIDEO_DECODE_ACCELERATOR_H_
+
+#include <map>
+#include <memory>
+
+#include "base/callback_forward.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/sequence_checker.h"
+#include "base/sequenced_task_runner.h"
+#include "base/time/time.h"
+#include "media/base/video_decoder.h"
+#include "media/gpu/chromeos/dmabuf_video_frame_pool.h"
+#include "media/gpu/chromeos/fourcc.h"
+#include "media/gpu/chromeos/vda_video_frame_pool.h"
+#include "media/gpu/chromeos/video_frame_converter.h"
+#include "media/gpu/media_gpu_export.h"
+#include "media/video/video_decode_accelerator.h"
+
+namespace gpu {
+class GpuMemoryBufferFactory;
+}  // namespace gpu
+
+namespace media {
+
+class VideoFrame;
+
+// Implements the VideoDecodeAccelerator backed by a VideoDecoder.
+// Currently GpuArcVideoDecodeAccelerator bridges the video codec from
+// ARC++ to a VDA instance. We plan to deprecate to VDA implementation at
+// chromium by new VD implementation. So we need the adapter between
+// these two interface.
+//
+// Important note: This adaptor is only used temporary as an intermediate step
+// while the GpuArcVideoDecodeAccelerator is being ported to the new VD
+// interface. This Adaptor will be deprecated soon and should not be used
+// anywhere else.
+class MEDIA_GPU_EXPORT VdVideoDecodeAccelerator
+    : public VideoDecodeAccelerator,
+      public VdaVideoFramePool::VdaDelegate {
+ public:
+  // Callback for creating VideoDecoder instance.
+  using CreateVideoDecoderCb =
+      base::RepeatingCallback<std::unique_ptr<VideoDecoder>(
+          scoped_refptr<base::SequencedTaskRunner>,
+          std::unique_ptr<DmabufVideoFramePool>,
+          std::unique_ptr<VideoFrameConverter>,
+          gpu::GpuMemoryBufferFactory* const)>;
+
+  // Create VdVideoDecodeAccelerator instance, and call Initialize().
+  // Return nullptr if Initialize() failed.
+  static std::unique_ptr<VideoDecodeAccelerator> Create(
+      CreateVideoDecoderCb create_vd_cb,
+      Client* client,
+      const Config& config,
+      scoped_refptr<base::SequencedTaskRunner> task_runner);
+  VdVideoDecodeAccelerator(const VdVideoDecodeAccelerator&) = delete;
+  VdVideoDecodeAccelerator& operator=(const VdVideoDecodeAccelerator&) = delete;
+  ~VdVideoDecodeAccelerator() override;
+
+  // Implementation of VideoDecodeAccelerator.
+  bool Initialize(const Config& config, Client* client) override;
+  void AssignPictureBuffers(const std::vector<PictureBuffer>& buffers) override;
+  void ImportBufferForPicture(
+      int32_t picture_buffer_id,
+      VideoPixelFormat pixel_format,
+      gfx::GpuMemoryBufferHandle gpu_memory_buffer_handle) override;
+  void ReusePictureBuffer(int32_t picture_buffer_id) override;
+  void Decode(BitstreamBuffer bitstream_buffer) override;
+  void Decode(scoped_refptr<DecoderBuffer> buffer,
+              int32_t bitstream_id) override;
+  void Flush() override;
+  void Reset() override;
+  void Destroy() override;
+
+  // Implementation of VdaVideoFramePool::VdaDelegate.
+  void RequestFrames(const Fourcc& fourcc,
+                     const gfx::Size& coded_size,
+                     const gfx::Rect& visible_rect,
+                     size_t max_num_frames,
+                     NotifyLayoutChangedCb notify_layout_changed_cb,
+                     ImportFrameCb import_frame_cb) override;
+
+ private:
+  using DmabufId = DmabufVideoFramePool::DmabufId;
+
+  VdVideoDecodeAccelerator(
+      CreateVideoDecoderCb create_vd_cb,
+      scoped_refptr<base::SequencedTaskRunner> task_runner);
+
+  // Callback methods of |vd_|.
+  void OnInitializeDone(bool success);
+  void OnDecodeDone(int32_t bitstream_buffer_id, DecodeStatus status);
+  void OnFrameReady(scoped_refptr<VideoFrame> frame);
+  void OnFlushDone(DecodeStatus status);
+  void OnResetDone();
+
+  // Get Picture instance that represents the same buffer as |frame|. Return
+  // base::nullopt if the buffer is already dismissed.
+  base::Optional<Picture> GetPicture(const VideoFrame& frame);
+
+  // Thunk to post OnFrameReleased() to |task_runner|.
+  // Because this thunk may be called in any thread, We don't want to
+  // dereference WeakPtr. Therefore we wrap the WeakPtr by base::Optional to
+  // avoid the task runner defererencing the WeakPtr.
+  static void OnFrameReleasedThunk(
+      base::Optional<base::WeakPtr<VdVideoDecodeAccelerator>> weak_this,
+      scoped_refptr<base::SequencedTaskRunner> task_runner,
+      scoped_refptr<VideoFrame> origin_frame);
+  // Called when a frame gets destroyed.
+  void OnFrameReleased(scoped_refptr<VideoFrame> origin_frame);
+
+  // Called when any error occurs. Notify |client_| an error occurred.
+  void OnError(base::Location location, Error error);
+
+  // Callback to generate VideoDecoder.
+  CreateVideoDecoderCb create_vd_cb_;
+  // The client of this VDA.
+  VideoDecodeAccelerator::Client* client_ = nullptr;
+  // The delegated VideoDecoder instance.
+  std::unique_ptr<VideoDecoder> vd_;
+  // Callback for returning the result after this instance is asked to request
+  // new frames. The VdaVideoFramePool is blocked until this callback is called.
+  NotifyLayoutChangedCb notify_layout_changed_cb_;
+  // Callback for passing the available frames to the pool.
+  ImportFrameCb import_frame_cb_;
+
+  // The size requested from VdaVideoFramePool.
+  gfx::Size pending_coded_size_;
+  // The formats of the current buffers.
+  gfx::Size coded_size_;
+  base::Optional<VideoFrameLayout> layout_;
+
+  // Mapping from VideoFrame's DmabufId to picture buffer id.
+  std::map<DmabufId, int32_t /* picture_buffer_id */> frame_id_to_picture_id_;
+  // Record how many times the picture is sent to the client, and keep a refptr
+  // of corresponding VideoFrame when the client owns the buffers.
+  std::map<int32_t /* picture_buffer_id */,
+           std::pair<scoped_refptr<VideoFrame>, size_t /* num_sent */>>
+      picture_at_client_;
+
+  // Main task runner and its sequence checker. All methods should be called
+  // on it.
+  scoped_refptr<base::SequencedTaskRunner> client_task_runner_;
+  SEQUENCE_CHECKER(client_sequence_checker_);
+
+  // The weak pointer of this class instance, bound to |client_task_runner_|.
+  base::WeakPtr<VdVideoDecodeAccelerator> weak_this_;
+  base::WeakPtrFactory<VdVideoDecodeAccelerator> weak_this_factory_{this};
+};
+
+}  // namespace media
+
+#endif  // MEDIA_GPU_CHROMEOS_VD_VIDEO_DECODE_ACCELERATOR_H_

media/gpu/chromeos/vda_video_frame_pool.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "media/gpu/chromeos/vda_video_frame_pool.h"
+
+#include "base/bind.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/synchronization/waitable_event.h"
+#include "media/gpu/chromeos/gpu_buffer_layout.h"
+#include "media/gpu/macros.h"
+
+namespace media {
+
+VdaVideoFramePool::VdaVideoFramePool(
+    base::WeakPtr<VdaDelegate> vda,
+    scoped_refptr<base::SequencedTaskRunner> vda_task_runner)
+    : vda_(std::move(vda)), vda_task_runner_(std::move(vda_task_runner)) {
+  DVLOGF(3);
+  DETACH_FROM_SEQUENCE(parent_sequence_checker_);
+
+  weak_this_ = weak_this_factory_.GetWeakPtr();
+}
+
+VdaVideoFramePool::~VdaVideoFramePool() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  weak_this_factory_.InvalidateWeakPtrs();
+}
+
+base::Optional<GpuBufferLayout> VdaVideoFramePool::Initialize(
+    const Fourcc& fourcc,
+    const gfx::Size& coded_size,
+    const gfx::Rect& visible_rect,
+    const gfx::Size& natural_size,
+    size_t max_num_frames) {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  visible_rect_ = visible_rect;
+  natural_size_ = natural_size;
+
+  if (max_num_frames_ == max_num_frames && fourcc_ && *fourcc_ == fourcc &&
+      coded_size_ == coded_size) {
+    DVLOGF(3) << "Arguments related to frame layout are not changed, skip.";
+    return layout_;
+  }
+  max_num_frames_ = max_num_frames;
+  fourcc_ = fourcc;
+  coded_size_ = coded_size;
+
+  // Clear the pool and reset the layout to prevent previous frames are recycled
+  // back to the pool.
+  frame_pool_ = {};
+  layout_ = base::nullopt;
+
+  // Receive the layout from the callback. |layout_| is accessed on
+  // |parent_task_runner_| except OnRequestFramesDone(). However, we block
+  // |parent_task_runner_| until OnRequestFramesDone() returns. So we don't need
+  // a lock to protect |layout_|.
+  // Also it's safe to use base::Unretained() here because we block here, |this|
+  // must be alive during the callback.
+  base::WaitableEvent done;
+  vda_task_runner_->PostTask(
+      FROM_HERE,
+      base::BindOnce(&VdaDelegate::RequestFrames, vda_, fourcc, coded_size,
+                     visible_rect, max_num_frames,
+                     base::BindOnce(&VdaVideoFramePool::OnRequestFramesDone,
+                                    base::Unretained(this), &done),
+                     base::BindRepeating(&VdaVideoFramePool::ImportFrameThunk,
+                                         parent_task_runner_, weak_this_)));
+  done.Wait();
+
+  return layout_;
+}
+
+void VdaVideoFramePool::OnRequestFramesDone(
+    base::WaitableEvent* done,
+    base::Optional<GpuBufferLayout> value) {
+  DVLOGF(3);
+  // RequestFrames() is blocked on |parent_task_runner_| to wait for this method
+  // finishes, so this method must not be run on the same sequence.
+  DCHECK(!parent_task_runner_->RunsTasksInCurrentSequence());
+
+  DCHECK(fourcc_);
+  DCHECK_EQ(value->fourcc(), *fourcc_);
+  DCHECK_GE(value->size().height(), coded_size_.height());
+  DCHECK_GE(value->size().width(), coded_size_.width());
+
+  layout_ = value;
+  done->Signal();
+}
+
+// static
+void VdaVideoFramePool::ImportFrameThunk(
+    scoped_refptr<base::SequencedTaskRunner> task_runner,
+    base::Optional<base::WeakPtr<VdaVideoFramePool>> weak_this,
+    scoped_refptr<VideoFrame> frame) {
+  DVLOGF(3);
+  DCHECK(weak_this);
+
+  task_runner->PostTask(
+      FROM_HERE, base::BindOnce(&VdaVideoFramePool::ImportFrame, *weak_this,
+                                std::move(frame)));
+}
+
+void VdaVideoFramePool::ImportFrame(scoped_refptr<VideoFrame> frame) {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  if (!layout_ || layout_->fourcc().ToVideoPixelFormat() != frame->format() ||
+      layout_->size() != frame->coded_size()) {
+    return;
+  }
+
+  frame_pool_.push(std::move(frame));
+  CallFrameAvailableCbIfNeeded();
+}
+
+scoped_refptr<VideoFrame> VdaVideoFramePool::GetFrame() {
+  DVLOGF(3);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  if (IsExhausted())
+    return nullptr;
+
+  scoped_refptr<VideoFrame> origin_frame = std::move(frame_pool_.front());
+  frame_pool_.pop();
+
+  // Update visible_rect and natural_size.
+  scoped_refptr<VideoFrame> wrapped_frame = VideoFrame::WrapVideoFrame(
+      origin_frame, origin_frame->format(), visible_rect_, natural_size_);
+  DCHECK(wrapped_frame);
+  wrapped_frame->AddDestructionObserver(
+      base::BindOnce(&VdaVideoFramePool::ImportFrameThunk, parent_task_runner_,
+                     weak_this_, std::move(origin_frame)));
+  return wrapped_frame;
+}
+
+bool VdaVideoFramePool::IsExhausted() {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  return frame_pool_.empty();
+}
+
+void VdaVideoFramePool::NotifyWhenFrameAvailable(base::OnceClosure cb) {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  frame_available_cb_ = std::move(cb);
+  CallFrameAvailableCbIfNeeded();
+}
+
+void VdaVideoFramePool::CallFrameAvailableCbIfNeeded() {
+  DVLOGF(4);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(parent_sequence_checker_);
+
+  if (frame_available_cb_ && !IsExhausted())
+    parent_task_runner_->PostTask(FROM_HERE, std::move(frame_available_cb_));
+}
+
+}  // namespace media

media/gpu/chromeos/vda_video_frame_pool.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef MEDIA_GPU_CHROMEOS_VDA_VIDEO_FRAME_POOL_H_
+#define MEDIA_GPU_CHROMEOS_VDA_VIDEO_FRAME_POOL_H_
+
+#include <vector>
+
+#include "base/containers/queue.h"
+#include "base/memory/scoped_refptr.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "base/sequenced_task_runner.h"
+#include "media/base/video_frame.h"
+#include "media/gpu/chromeos/dmabuf_video_frame_pool.h"
+#include "media/gpu/chromeos/fourcc.h"
+
+namespace base {
+class WaitableEvent;
+}
+
+namespace media {
+
+class GpuBufferLayout;
+
+// This class is used by VdVideoDecodeAccelerator, which adapts
+// VideoDecodeAccelerator to VideoDecoder interface.
+// The mission is to allocate and manage DMA-buf VideoFrame by delegating the
+// requests of buffer allocation to a VideoDecodeAccelerator instance, and
+// provide VideoFrame to the VideoDecoder instance.
+// The communication with VdVideoDecodeAccelerator, which inherits
+// VdaDelegate, is executed on |vda_task_runner_|, and the communication with
+// VideoDecoder instance is on |parent_task_runner_|.
+class VdaVideoFramePool : public DmabufVideoFramePool {
+ public:
+  class VdaDelegate {
+   public:
+    // Callback for returning the layout of requested buffer.
+    using NotifyLayoutChangedCb =
+        base::OnceCallback<void(base::Optional<GpuBufferLayout>)>;
+    // Callback for importing available frames to this pool.
+    using ImportFrameCb =
+        base::RepeatingCallback<void(scoped_refptr<VideoFrame>)>;
+
+    // Request new frames from VDA's client. VdaDelegate has to return the
+    // layout of frames by calling |notify_layout_changed_cb|.
+    // After that, VdaDelegate should pass frames by calling
+    // |import_frame_cb|.
+    // Note: RequestFrames(), |notify_layout_changed_cb|, and |import_frame_cb|
+    // should be called on VdaVideoFramePool's |vda_task_runner_|.
+    virtual void RequestFrames(const Fourcc& fourcc,
+                               const gfx::Size& coded_size,
+                               const gfx::Rect& visible_rect,
+                               size_t max_num_frames,
+                               NotifyLayoutChangedCb notify_layout_changed_cb,
+                               ImportFrameCb import_frame_cb) = 0;
+  };
+
+  VdaVideoFramePool(base::WeakPtr<VdaDelegate> vda,
+                    scoped_refptr<base::SequencedTaskRunner> vda_task_runner);
+  ~VdaVideoFramePool() override;
+
+  // DmabufVideoFramePool implementation.
+  base::Optional<GpuBufferLayout> Initialize(const Fourcc& fourcc,
+                                             const gfx::Size& coded_size,
+                                             const gfx::Rect& visible_rect,
+                                             const gfx::Size& natural_size,
+                                             size_t max_num_frames) override;
+  scoped_refptr<VideoFrame> GetFrame() override;
+  bool IsExhausted() override;
+  void NotifyWhenFrameAvailable(base::OnceClosure cb) override;
+
+ private:
+  // Update the layout of the buffers. |vda_| calls this as
+  // NotifyLayoutChangedCb.
+  void OnRequestFramesDone(base::WaitableEvent* done,
+                           base::Optional<GpuBufferLayout> value);
+
+  // Thunk to post ImportFrame() to |task_runner|.
+  // Because this thunk may be called in any thread, We don't want to
+  // dereference WeakPtr. Therefore we wrap the WeakPtr by base::Optional to
+  // avoid the task runner defererencing the WeakPtr.
+  static void ImportFrameThunk(
+      scoped_refptr<base::SequencedTaskRunner> task_runner,
+      base::Optional<base::WeakPtr<VdaVideoFramePool>> weak_this,
+      scoped_refptr<VideoFrame> frame);
+  // Import an available frame.
+  void ImportFrame(scoped_refptr<VideoFrame> frame);
+
+  // Call |frame_available_cb_| when the pool is not exhausted.
+  void CallFrameAvailableCbIfNeeded();
+
+  // WeakPtr of VdaDelegate instance, bound at |vda_task_runner_|.
+  base::WeakPtr<VdaDelegate> vda_;
+  // Task runner that interacts with VdaDelegate. All VdaDelegate's methods
+  // and their callbacks should be called on this task runner.
+  // Note: DmabufVideoFrame's public methods like Initialize() and GetFrame()
+  // should be called on |parent_task_runner_|.
+  scoped_refptr<base::SequencedTaskRunner> vda_task_runner_;
+
+  // The callback which is called when the pool is not exhausted.
+  base::OnceClosure frame_available_cb_;
+
+  // The layout of the frames in |frame_pool_|.
+  base::Optional<GpuBufferLayout> layout_;
+
+  // Data passed from Initialize().
+  size_t max_num_frames_ = 0;
+  base::Optional<Fourcc> fourcc_;
+  gfx::Size coded_size_;
+  gfx::Rect visible_rect_;
+  gfx::Size natural_size_;
+
+  base::queue<scoped_refptr<VideoFrame>> frame_pool_;
+
+  // Sequence checker for |vda_task_runner_|.
+  SEQUENCE_CHECKER(vda_sequence_checker_);
+  // Sequence checker for |parent_task_runner_|.
+  SEQUENCE_CHECKER(parent_sequence_checker_);
+
+  // The weak pointer of this, bound at |parent_task_runner_|.
+  base::WeakPtr<VdaVideoFramePool> weak_this_;
+  base::WeakPtrFactory<VdaVideoFramePool> weak_this_factory_{this};
+};
+
+}  // namespace media
+
+#endif  // MEDIA_GPU_CHROMEOS_VDA_VIDEO_FRAME_POOL_H_