[webcodecs] Remove Streams from VideoDecoder

Change-Id: If11171fae55dc0b4fa492908884ab48d1805b18c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2092692
Commit-Queue: Dan Sanders <sandersd@chromium.org>
Reviewed-by: Nate Chapin <japhet@chromium.org>
Reviewed-by: Chrome Cunningham <chcunningham@chromium.org>
Cr-Commit-Position: refs/heads/master@{#755992}

third_party/blink/renderer/bindings/idl_in_modules.gni

@@ -723,10 +723,13 @@ static_idl_files_in_modules = get_path_info(
           "//third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.idl",
           "//third_party/blink/renderer/modules/webcodecs/encoded_video_config.idl",
           "//third_party/blink/renderer/modules/webcodecs/video_decoder.idl",
+          "//third_party/blink/renderer/modules/webcodecs/video_decoder_init.idl",
+          "//third_party/blink/renderer/modules/webcodecs/video_decoder_output_callback.idl",
           "//third_party/blink/renderer/modules/webcodecs/video_frame.idl",
           "//third_party/blink/renderer/modules/webcodecs/video_track_reader.idl",
           "//third_party/blink/renderer/modules/webcodecs/video_track_writer.idl",
           "//third_party/blink/renderer/modules/webcodecs/video_track_writer_parameters.idl",
+          "//third_party/blink/renderer/modules/webcodecs/webcodecs_error_callback.idl",
           "//third_party/blink/renderer/modules/webdatabase/database.idl",
           "//third_party/blink/renderer/modules/webdatabase/sql_error.idl",
           "//third_party/blink/renderer/modules/webdatabase/sql_result_set.idl",

third_party/blink/renderer/bindings/modules/v8/generated.gni

@@ -160,8 +160,12 @@ generated_modules_callback_function_files = [
   "$bindings_modules_v8_output_dir/v8_storage_quota_callback.h",
   "$bindings_modules_v8_output_dir/v8_storage_usage_callback.cc",
   "$bindings_modules_v8_output_dir/v8_storage_usage_callback.h",
+  "$bindings_modules_v8_output_dir/v8_video_decoder_output_callback.cc",
+  "$bindings_modules_v8_output_dir/v8_video_decoder_output_callback.h",
   "$bindings_modules_v8_output_dir/v8_video_frame_request_callback.cc",
   "$bindings_modules_v8_output_dir/v8_video_frame_request_callback.h",
+  "$bindings_modules_v8_output_dir/v8_web_codecs_error_callback.cc",
+  "$bindings_modules_v8_output_dir/v8_web_codecs_error_callback.h",
   "$bindings_modules_v8_output_dir/v8_xr_frame_request_callback.cc",
   "$bindings_modules_v8_output_dir/v8_xr_frame_request_callback.h",
 ]

third_party/blink/renderer/modules/webcodecs/idls.gni

@@ -10,8 +10,14 @@ modules_idl_files = [
   "video_track_writer.idl",
 ]
 
+modules_callback_function_idl_files = [
+  "video_decoder_output_callback.idl",
+  "webcodecs_error_callback.idl",
+]
+
 modules_dictionary_idl_files = [
   "encoded_video_config.idl",
+  "video_decoder_init.idl",
   "video_track_writer_parameters.idl",
 ]
 

third_party/blink/renderer/modules/webcodecs/video_decoder.h

@@ -5,87 +5,84 @@
 #ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_H_
 #define THIRD_PARTY_BLINK_RENDERER_MODULES_WEBCODECS_VIDEO_DECODER_H_
 
+#include <stdint.h>
 #include <memory>
 
-#include "base/memory/weak_ptr.h"
-#include "base/sequence_checker.h"
 #include "media/base/status.h"
 #include "media/base/video_decoder.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
+#include "third_party/blink/renderer/platform/heap/garbage_collected.h"
+#include "third_party/blink/renderer/platform/heap/heap.h"
+#include "third_party/blink/renderer/platform/heap/heap_allocator.h"
+#include "third_party/blink/renderer/platform/heap/member.h"
 
 namespace blink {
 
+class EncodedVideoChunk;
 class EncodedVideoConfig;
-class ReadableStream;
-class WritableStream;
+class ExceptionState;
+class ScriptState;
+class VideoDecoderInit;
 
 class MODULES_EXPORT VideoDecoder final : public ScriptWrappable {
   DEFINE_WRAPPERTYPEINFO();
 
  public:
-  static VideoDecoder* Create(ScriptState*);
+  static VideoDecoder* Create(ScriptState*,
+                              const VideoDecoderInit*,
+                              ExceptionState&);
 
-  VideoDecoder(ScriptState*);
+  VideoDecoder(ScriptState*, const VideoDecoderInit*, ExceptionState&);
   ~VideoDecoder() override;
 
   // video_decoder.idl implementation.
-  ScriptPromise configure(const EncodedVideoConfig* config,
-                          ExceptionState& exception_state);
-  ReadableStream* readable() const;
-  WritableStream* writable() const;
+  int32_t decodeQueueSize();
+  int32_t decodeProcessingCount();
+  ScriptPromise configure(const EncodedVideoConfig*, ExceptionState&);
+  ScriptPromise decode(const EncodedVideoChunk*, ExceptionState&);
+  ScriptPromise flush(ExceptionState&);
+  ScriptPromise reset(ExceptionState&);
 
   // GarbageCollected override.
   void Trace(Visitor*) override;
 
  private:
-  SEQUENCE_CHECKER(sequence_checker_);
-
-  class UnderlyingSink;
-  class UnderlyingSource;
-
-  // Creates a new |write_resolver_| and returns a promise from it.
-  ScriptPromise CreateWritePromise();
-
-  // Resolves |write_resolver_| if the current state can accept a write.
-  void MaybeAcceptWrite();
-
-  // Rejects promises and shuts down streams.
+  struct Request : public GarbageCollected<Request> {
+    enum class Type {
+      kConfigure,
+      kDecode,
+      kFlush,
+      kReset,
+    };
+
+    void Trace(Visitor*);
+
+    Type type;
+    Member<const EncodedVideoConfig> config;
+    Member<const EncodedVideoChunk> chunk;
+    Member<ScriptPromiseResolver> resolver;
+  };
+
+  ScriptPromise EnqueueRequest(Request* request);
+  void ProcessRequests();
   void HandleError();
 
   // Called by |decoder_|.
   void OnInitializeDone(media::Status status);
-  void OnDecodeDone(media::DecodeStatus);
+  void OnDecodeDone(uint32_t id, media::DecodeStatus);
   void OnOutput(scoped_refptr<media::VideoFrame>);
 
-  // Called by UnderlyingSink.
-  ScriptPromise Start(ExceptionState&);
-  ScriptPromise Write(ScriptValue chunk, ExceptionState&);
-  ScriptPromise Close(ExceptionState&);
-  ScriptPromise Abort(ExceptionState&);
-
-  // Called by UnderlyingSource.
-  ScriptPromise Pull();
-  ScriptPromise Cancel();
-
   Member<ScriptState> script_state_;
 
-  Member<UnderlyingSink> underlying_sink_;
-  Member<WritableStream> writable_;
-  Member<UnderlyingSource> underlying_source_;
-  Member<ReadableStream> readable_;
-
-  // Signals completion of configure().
-  Member<ScriptPromiseResolver> configure_resolver_;
-
-  // Signals ability to accept an input chunk.
-  Member<ScriptPromiseResolver> write_resolver_;
+  HeapDeque<Member<Request>> requests_;
+  int32_t requested_decodes_ = 0;
+  int32_t requested_resets_ = 0;
 
   std::unique_ptr<media::VideoDecoder> decoder_;
-  bool has_error_ = false;
-  int pending_decodes_ = 0;
+  HeapHashMap<uint32_t, Member<Request>> pending_decodes_;
 };
 
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/video_decoder.idl

@@ -4,23 +4,84 @@
 
 // https://github.com/WICG/web-codecs
 
+// A VideoDecoder processes a queue of configure, decode, and flush requests.
+// Requests are taken from the queue sequentially but may be processed
+// concurrently.
+//
+// TODO(sandersd): Specify a tune() implementation for changing decoder
+// parameters (separate from stream parameters). This is more important for
+// encoders.
 [RuntimeEnabled=WebCodecs]
 interface VideoDecoder {
-  // TODO(sandersd): Configure properties of streams (type and high water mark).
-  [CallWith=ScriptState] constructor();
+  // |init| includes an |output| callback for emitting VideoFrames and an
+  // |error| callback for emitting decode errors.
+  //
+  // When in an error state, methods other than reset() will fail.
+  //
+  // TODO(sandersd): Consider adding a state or last error attribute.
+  // TODO(sandersd): Consider aborting pending decodes on error, rather than
+  // waiting for reset().
+  [CallWith=ScriptState, RaisesException] constructor(VideoDecoderInit init);
+
+  // The number of queued decode requests. This does not include requests that
+  // have been taken for processing.
+  //
+  // Applications can minimize underflow by enqueueing decode requests until
+  // |decodeQueueSize| is greater than a constant.
+  readonly attribute long decodeQueueSize;
+
+  // The number of decode requests currently being processed.
+  //
+  // Applications can minimize resource consumption and decode latency by
+  // enqueueing decode requests only when |decodeQueueSize| and
+  // |decodeProcessingCount| are small.
+  //
+  // TODO(sandersd): Consider counting queued decode requests as well. This
+  // could be simpler for apps.
+  readonly attribute long decodeProcessingCount;
 
-  // Configure or reconfigure the decoder.
+  // Enqueue a request to set or change the stream configuration.
+  //
+  // The next enqueued decode request must be for a keyframe.
   //
-  // |writable| and |readable| will be errored and set to null.
+  // Resolved after emitting output for all earlier decode requests.
   //
-  // If configuration is successful, |writable| and |readable| will be set to
-  // new streams.
+  // TODO(sandersd): Test that resolution (a microtask) interleaves between
+  // outputs callback calls in all cases.
+  // TODO(sandersd): Move the keyframe rule into the bytestream registry.
   [RaisesException] Promise<void> configure(EncodedVideoConfig config);
 
-  // Closing |writable| will flush the decoder, then close |readable|.
-  // Aborting |writable| will error |readable|.
-  readonly attribute WritableStream? writable;  // of EncodedVideoChunk
+  // Enqueue a request to decode an input chunk.
+  //
+  // You must call configure() before calling enqueue() for the first time.
+  //
+  // Resolved after decoding of the input chunk has started (that is, after
+  // decreasing |decodeQueueSize|).
+  //
+  // TODO(sandersd): Change to a dictionary type.
+  // TODO(sandersd): Should we guarantee that resolution occurs in order?
+  // TODO(sandersd): Add status to result.
+  // TODO(sandersd): Buffer return.
+  [RaisesException] Promise<void> decode(EncodedVideoChunk chunk);
+
+  // Enqueue a request to finish decoding queued input chunks.
+  //
+  // The next enqueued input chunk must be a keyframe.
+  //
+  // Resolved after emitting output for all earlier decode requests.
+  //
+  // TODO(sandersd): Consider relaxing the keyframe requirement.
+  [RaisesException] Promise<void> flush();
 
-  // Canceling |readable| will error |writable|.
-  readonly attribute ReadableStream? readable;  // of VideoFrame
+  // Discard all pending work.
+  //
+  // Output for earlier decode requests will not be emitted, even if processing
+  // has already started.
+  //
+  // The next enqueued input chunk must be a keyframe.
+  //
+  // Resolved after all earlier enqueue() promises have been resolved.
+  //
+  // TODO(sandersd): Require configure() after reset()?
+  [RaisesException] Promise<void> reset();
 };

third_party/blink/renderer/modules/webcodecs/video_decoder_init.idl

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/WICG/web-codecs
+
+dictionary VideoDecoderInit {
+  VideoDecoderOutputCallback output;
+  WebCodecsErrorCallback error;
+};

third_party/blink/renderer/modules/webcodecs/video_decoder_output_callback.idl

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/WICG/web-codecs
+
+[RuntimeEnabled=WebCodecs]
+callback VideoDecoderOutputCallback = void(VideoFrame output);

third_party/blink/renderer/modules/webcodecs/webcodecs_error_callback.idl

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// https://github.com/WICG/web-codecs
+
+[RuntimeEnabled=WebCodecs]
+callback WebCodecsErrorCallback = void(DOMException error);

third_party/blink/web_tests/webexposed/global-interface-listing-expected.txt

@@ -8718,10 +8718,13 @@ interface ValidityState
     method constructor
 interface VideoDecoder
     attribute @@toStringTag
-    getter readable
-    getter writable
+    getter decodeProcessingCount
+    getter decodeQueueSize
     method configure
     method constructor
+    method decode
+    method flush
+    method reset
 interface VideoFrame
     attribute @@toStringTag
     getter coded_height