webcodecs: Connect AudioEncoder to underlying encoder implementation

At this point only Opus encoding is supported.

Bug: 1094181, 1094179
Change-Id: I05b22bcbd29e96b3823d00dbf85f007936678751
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2625822Reviewed-by: Dan Sanders <sandersd@chromium.org>
Commit-Queue: Eugene Zemtsov <eugene@chromium.org>
Cr-Commit-Position: refs/heads/master@{#843360}

third_party/blink/renderer/modules/webcodecs/DEPS

@@ -8,6 +8,7 @@ include_rules = [
     "+gpu/command_buffer/client/raster_interface.h",
     "+gpu/GLES2/gl2extchromium.h",
 
+    "+media/audio",
     "+media/base",
     "+media/filters",
     "+media/formats/mp4/box_definitions.h",

third_party/blink/renderer/modules/webcodecs/audio_encoder.cc

@@ -4,8 +4,22 @@
 
 #include "third_party/blink/renderer/modules/webcodecs/audio_encoder.h"
 
+#include "media/audio/audio_opus_encoder.h"
+#include "media/base/audio_parameters.h"
+#include "third_party/blink/public/mojom/web_feature/web_feature.mojom-blink.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_audio_encoder_config.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_audio_frame_init.h"
+#include "third_party/blink/renderer/modules/webaudio/audio_buffer.h"
+#include "third_party/blink/renderer/modules/webcodecs/encoded_audio_chunk.h"
+#include "third_party/blink/renderer/modules/webcodecs/encoded_audio_metadata.h"
+
 namespace blink {
 
+// static
+const char* AudioEncoderTraits::GetNameForDevTools() {
+  return "AudioEncoder(WebCodecs)";
+}
+
 AudioEncoder* AudioEncoder::Create(ScriptState* script_state,
                                    const AudioEncoderInit* init,
                                    ExceptionState& exception_state) {
@@ -16,35 +30,178 @@ AudioEncoder* AudioEncoder::Create(ScriptState* script_state,
 AudioEncoder::AudioEncoder(ScriptState* script_state,
                            const AudioEncoderInit* init,
                            ExceptionState& exception_state)
-    : ExecutionContextLifecycleObserver(ExecutionContext::From(script_state)) {}
+    : Base(script_state, init, exception_state) {
+  UseCounter::Count(ExecutionContext::From(script_state),
+                    WebFeature::kWebCodecs);
+}
 
 AudioEncoder::~AudioEncoder() = default;
 
-int32_t AudioEncoder::encodeQueueSize() {
-  return 0;
+void AudioEncoder::ProcessConfigure(Request* request) {
+  DCHECK_NE(state_.AsEnum(), V8CodecState::Enum::kClosed);
+  DCHECK_EQ(request->type, Request::Type::kConfigure);
+  DCHECK(active_config_);
+  DCHECK_EQ(active_config_->codec, media::kCodecOpus);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+
+  auto output_cb = WTF::BindRepeating(
+      &AudioEncoder::CallOutputCallback, WrapCrossThreadWeakPersistent(this),
+      // We can't use |active_config_| from |this| because it can change by
+      // the time the callback is executed.
+      WrapCrossThreadPersistent(active_config_.Get()), reset_count_);
+
+  auto status_callback = [](AudioEncoder* self, uint32_t reset_count,
+                            media::Status status) {
+    if (!self || self->reset_count_ != reset_count)
+      return;
+    DCHECK_CALLED_ON_VALID_SEQUENCE(self->sequence_checker_);
+    if (!status.is_ok()) {
+      self->HandleError(
+          self->logger_->MakeException("Encoding error.", status));
+    }
+  };
+
+  media::AudioParameters input_params(media::AudioParameters::AUDIO_PCM_LINEAR,
+                                      media::CHANNEL_LAYOUT_DISCRETE,
+                                      active_config_->sample_rate, 0);
+  input_params.set_channels_for_discrete(active_config_->channels);
+  media_encoder_ = std::make_unique<media::AudioOpusEncoder>(
+      input_params, output_cb,
+      WTF::BindRepeating(status_callback, WrapCrossThreadWeakPersistent(this),
+                         reset_count_),
+      active_config_->bitrate);
 }
 
-void AudioEncoder::encode(AudioFrame* frame, ExceptionState&) {}
+void AudioEncoder::ProcessEncode(Request* request) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  DCHECK_EQ(state_, V8CodecState::Enum::kConfigured);
+  DCHECK(media_encoder_);
+  DCHECK_EQ(request->type, Request::Type::kEncode);
+  DCHECK_GT(requested_encodes_, 0);
 
-void AudioEncoder::configure(const AudioEncoderConfig*, ExceptionState&) {}
+  auto* frame = request->frame.Release();
 
-ScriptPromise AudioEncoder::flush(ExceptionState&) {
-  return ScriptPromise();
+  base::TimeTicks time =
+      base::TimeTicks() + base::TimeDelta::FromMicroseconds(frame->timestamp());
+
+  auto* buffer = frame->buffer();
+  DCHECK(buffer);
+  {
+    auto audio_bus = media::AudioBus::CreateWrapper(buffer->numberOfChannels());
+    for (int channel = 0; channel < audio_bus->channels(); channel++) {
+      float* data = buffer->getChannelData(channel)->Data();
+      DCHECK(data);
+      audio_bus->SetChannelData(channel, data);
+    }
+    audio_bus->set_frames(buffer->length());
+    media_encoder_->EncodeAudio(*audio_bus, time);
+  }
+
+  frame->destroy();
+}
+
+void AudioEncoder::ProcessReconfigure(Request* request) {
+  // Audio decoders don't currently support any meaningful reconfiguring
 }
+void AudioEncoder::ProcessFlush(Request* request) {
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  DCHECK_EQ(state_, V8CodecState::Enum::kConfigured);
+  DCHECK(media_encoder_);
+  DCHECK_EQ(request->type, Request::Type::kFlush);
 
-void AudioEncoder::reset(ExceptionState&) {}
+  media_encoder_->Flush();
+  request->resolver->Resolve();
+}
 
-void AudioEncoder::close(ExceptionState&) {}
+AudioEncoder::ParsedConfig* AudioEncoder::ParseConfig(
+    const AudioEncoderConfig* opts,
+    ExceptionState& exception_state) {
+  auto* result = MakeGarbageCollected<ParsedConfig>();
+  result->codec = opts->codec().Utf8() == "opus" ? media::kCodecOpus
+                                                 : media::kUnknownAudioCodec;
+  result->channels = opts->numberOfChannels();
+  result->bitrate = opts->bitrate();
+  result->sample_rate = opts->sampleRate();
 
-String AudioEncoder::state() {
-  return "";
+  if (result->channels == 0) {
+    exception_state.ThrowTypeError("Invalid channel number.");
+    return nullptr;
+  }
+
+  if (result->bitrate == 0) {
+    exception_state.ThrowTypeError("Invalid bitrate.");
+    return nullptr;
+  }
+
+  if (result->sample_rate == 0) {
+    exception_state.ThrowTypeError("Invalid sample rate.");
+    return nullptr;
+  }
+  return result;
+}
+
+bool AudioEncoder::CanReconfigure(ParsedConfig& original_config,
+                                  ParsedConfig& new_config) {
+  return original_config.codec == new_config.codec &&
+         original_config.channels == new_config.channels &&
+         original_config.bitrate == new_config.bitrate &&
+         original_config.sample_rate == new_config.sample_rate;
+}
+
+AudioFrame* AudioEncoder::CloneFrame(AudioFrame* frame,
+                                     ExecutionContext* context) {
+  auto* init = AudioFrameInit::Create();
+  init->setTimestamp(frame->timestamp());
+
+  auto* buffer = frame->buffer();
+  if (!buffer)
+    return nullptr;
+
+  // Validata that buffer's data is consistent
+  for (auto channel = 0u; channel < buffer->numberOfChannels(); channel++) {
+    auto array = buffer->getChannelData(channel);
+    float* data = array->Data();
+    if (!data)
+      return nullptr;
+    if (array->length() != buffer->length())
+      return nullptr;
+  }
+
+  init->setBuffer(buffer);
+  return MakeGarbageCollected<AudioFrame>(init);
+}
+
+bool AudioEncoder::VerifyCodecSupport(ParsedConfig* config,
+                                      ExceptionState& exception_state) {
+  if (config->codec != media::kCodecOpus) {
+    exception_state.ThrowDOMException(DOMExceptionCode::kNotSupportedError,
+                                      "Unsupported codec type.");
+    return false;
+  }
+  return true;
 }
 
-void AudioEncoder::ContextDestroyed() {}
+void AudioEncoder::CallOutputCallback(
+    ParsedConfig* active_config,
+    uint32_t reset_count,
+    media::EncodedAudioBuffer encoded_buffer) {
+  if (!script_state_->ContextIsValid() || !output_callback_ ||
+      state_.AsEnum() != V8CodecState::Enum::kConfigured ||
+      reset_count != reset_count_)
+    return;
 
-void AudioEncoder::Trace(Visitor* visitor) const {
-  ScriptWrappable::Trace(visitor);
-  ExecutionContextLifecycleObserver::Trace(visitor);
+  DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
+  EncodedAudioMetadata metadata;
+  metadata.timestamp = encoded_buffer.timestamp - base::TimeTicks();
+  auto deleter = [](void* data, size_t length, void*) {
+    delete[] static_cast<uint8_t*>(data);
+  };
+  ArrayBufferContents data(encoded_buffer.encoded_data.release(),
+                           encoded_buffer.encoded_data_size, deleter);
+  auto* dom_array = MakeGarbageCollected<DOMArrayBuffer>(std::move(data));
+  auto* chunk = MakeGarbageCollected<EncodedAudioChunk>(metadata, dom_array);
+  ScriptState::Scope scope(script_state_);
+  output_callback_->InvokeAndReportException(nullptr, chunk);
 }
 
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/audio_encoder.h

@@ -7,14 +7,16 @@
 
 #include <memory>
 
+#include "media/base/audio_codecs.h"
+#include "media/base/audio_encoder.h"
 #include "third_party/blink/renderer/bindings/core/v8/active_script_wrappable.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise_resolver.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_codec_state.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_encoded_audio_chunk_output_callback.h"
-#include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_output_callback.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_webcodecs_error_callback.h"
 #include "third_party/blink/renderer/modules/modules_export.h"
 #include "third_party/blink/renderer/modules/webcodecs/audio_frame.h"
+#include "third_party/blink/renderer/modules/webcodecs/encoder_base.h"
 #include "third_party/blink/renderer/platform/bindings/script_wrappable.h"
 
 namespace blink {
@@ -23,10 +25,37 @@ class ExceptionState;
 class AudioEncoderConfig;
 class AudioEncoderInit;
 
+class MODULES_EXPORT AudioEncoderTraits {
+ public:
+  struct ParsedConfig final : public GarbageCollected<ParsedConfig> {
+    media::AudioCodec codec = media::kUnknownAudioCodec;
+    int channels = 0;
+    uint64_t bitrate = 0;
+    uint32_t sample_rate = 0;
+
+    void Trace(Visitor*) const {}
+  };
+
+  struct AudioEncoderEncodeOptions
+      : public GarbageCollected<AudioEncoderEncodeOptions> {
+    void Trace(Visitor*) const {}
+  };
+
+  using Init = AudioEncoderInit;
+  using Config = AudioEncoderConfig;
+  using InternalConfig = ParsedConfig;
+  using Frame = AudioFrame;
+  using EncodeOptions = AudioEncoderEncodeOptions;
+  using OutputChunk = EncodedAudioChunk;
+  using OutputCallback = V8EncodedAudioChunkOutputCallback;
+  using MediaEncoder = media::AudioEncoder;
+
+  // Can't be a virtual method, because it's used from base ctor.
+  static const char* GetNameForDevTools();
+};
+
 class MODULES_EXPORT AudioEncoder final
-    : public ScriptWrappable,
-      public ActiveScriptWrappable<AudioEncoder>,
-      public ExecutionContextLifecycleObserver {
+    : public EncoderBase<AudioEncoderTraits> {
   DEFINE_WRAPPERTYPEINFO();
 
  public:
@@ -36,29 +65,30 @@ class MODULES_EXPORT AudioEncoder final
   AudioEncoder(ScriptState*, const AudioEncoderInit*, ExceptionState&);
   ~AudioEncoder() override;
 
-  // audio_encoder.idl implementation.
-  int32_t encodeQueueSize();
-
-  void encode(AudioFrame* frame, ExceptionState&);
-
-  void configure(const AudioEncoderConfig*, ExceptionState&);
-
-  ScriptPromise flush(ExceptionState&);
-
-  void reset(ExceptionState&);
+  void encode(AudioFrame* frame, ExceptionState& exception_state) {
+    return Base::encode(frame, nullptr, exception_state);
+  }
 
-  void close(ExceptionState&);
+ private:
+  using Base = EncoderBase<AudioEncoderTraits>;
+  using ParsedConfig = AudioEncoderTraits::ParsedConfig;
 
-  String state();
+  void ProcessEncode(Request* request) override;
+  void ProcessConfigure(Request* request) override;
+  void ProcessReconfigure(Request* request) override;
+  void ProcessFlush(Request* request) override;
 
-  // ExecutionContextLifecycleObserver override.
-  void ContextDestroyed() override;
+  ParsedConfig* ParseConfig(const AudioEncoderConfig* opts,
+                            ExceptionState&) override;
+  bool VerifyCodecSupport(ParsedConfig*, ExceptionState&) override;
+  AudioFrame* CloneFrame(AudioFrame*, ExecutionContext*) override;
 
-  // ScriptWrappable override.
-  bool HasPendingActivity() const override { return false; }
+  bool CanReconfigure(ParsedConfig& original_config,
+                      ParsedConfig& new_config) override;
 
-  // GarbageCollected override.
-  void Trace(Visitor*) const override;
+  void CallOutputCallback(ParsedConfig* active_config,
+                          uint32_t reset_count,
+                          media::EncodedAudioBuffer encoded_buffer);
 };
 
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/audio_encoder_config.idl

@@ -12,5 +12,7 @@ dictionary AudioEncoderConfig {
   required unsigned long sampleRate;
 
   // 1, 2, etc.
-  required unsigned long numberOfChannels;
+  required unsigned short numberOfChannels;
+
+  required unsigned long long bitrate;
 };

third_party/blink/renderer/modules/webcodecs/audio_frame.h

@@ -30,6 +30,8 @@ class MODULES_EXPORT AudioFrame final : public ScriptWrappable {
   uint64_t timestamp() const;
   AudioBuffer* buffer() const;
 
+  void destroy() { close(); }
+
   // GarbageCollected override.
   void Trace(Visitor*) const override;
 

third_party/blink/renderer/modules/webcodecs/encoder_base.cc

@@ -16,10 +16,12 @@
 #include "third_party/blink/renderer/bindings/core/v8/script_function.h"
 #include "third_party/blink/renderer/bindings/core/v8/script_promise.h"
 #include "third_party/blink/renderer/bindings/core/v8/v8_dom_exception.h"
+#include "third_party/blink/renderer/bindings/modules/v8/v8_audio_encoder_init.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_config.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_encode_options.h"
 #include "third_party/blink/renderer/bindings/modules/v8/v8_video_encoder_init.h"
 #include "third_party/blink/renderer/core/dom/dom_exception.h"
+#include "third_party/blink/renderer/modules/webcodecs/audio_encoder.h"
 #include "third_party/blink/renderer/modules/webcodecs/codec_state_helper.h"
 #include "third_party/blink/renderer/modules/webcodecs/encoded_video_chunk.h"
 #include "third_party/blink/renderer/modules/webcodecs/encoded_video_metadata.h"
@@ -67,11 +69,6 @@ EncoderBase<Traits>::~EncoderBase() {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
 }
 
-template <typename Traits>
-int32_t EncoderBase<Traits>::encodeQueueSize() {
-  return requested_encodes_;
-}
-
 template <typename Traits>
 void EncoderBase<Traits>::configure(const ConfigType* config,
                                     ExceptionState& exception_state) {
@@ -292,5 +289,6 @@ void EncoderBase<Traits>::Request::Trace(Visitor* visitor) const {
 }
 
 template class EncoderBase<VideoEncoderTraits>;
+template class EncoderBase<AudioEncoderTraits>;
 
 }  // namespace blink

third_party/blink/renderer/modules/webcodecs/encoder_base.h

@@ -43,21 +43,21 @@ class MODULES_EXPORT EncoderBase
   ~EncoderBase() override;
 
   // *_encoder.idl implementation.
-  virtual int32_t encodeQueueSize();
+  int32_t encodeQueueSize() { return requested_encodes_; }
 
-  virtual void configure(const ConfigType*, ExceptionState&);
+  void configure(const ConfigType*, ExceptionState&);
 
-  virtual void encode(FrameType* frame,
+  void encode(FrameType* frame,
               const EncodeOptionsType* opts,
               ExceptionState& exception_state);
 
-  virtual ScriptPromise flush(ExceptionState&);
+  ScriptPromise flush(ExceptionState&);
 
-  virtual void reset(ExceptionState&);
+  void reset(ExceptionState&);
 
-  virtual void close(ExceptionState&);
+  void close(ExceptionState&);
 
-  virtual String state() { return state_; }
+  String state() { return state_; }
 
   // ExecutionContextLifecycleObserver override.
   void ContextDestroyed() override;

third_party/blink/web_tests/external/wpt/webcodecs/audio-encoder.any.js

+// META: global=window
+// META: script=/webcodecs/utils.js
+
+function make_audio_frame(timestamp, channels, sampleRate, length) {
+  let buffer = new AudioBuffer({
+    length: length,
+    numberOfChannels: channels,
+    sampleRate: sampleRate
+  });
+
+  for (var channel = 0; channel < buffer.numberOfChannels; channel++) {
+    // This gives us the actual array that contains the data
+    var array = buffer.getChannelData(channel);
+    let hz = 100 + channel * 50; // sound frequency
+    for (var i = 0; i < array.length; i++) {
+      let t = (i / sampleRate) * hz * (Math.PI * 2);
+      array[i] = Math.sin(t);
+    }
+  }
+
+  return new AudioFrame({
+    timestamp: timestamp,
+    buffer: buffer
+  });
+}
+
+promise_test(async t => {
+  let frame_count = 100;
+  let outputs = [];
+  let init = {
+    error: e => {
+      assert_unreached("error: " + e);
+    },
+    output: chunk => {
+      outputs.push(chunk);
+    }
+  };
+
+  let encoder = new AudioEncoder(init);
+
+  assert_equals(encoder.state, "unconfigured");
+  let config = {
+    codec: 'opus',
+    sampleRate: 48000,
+    numberOfChannels: 2,
+    bitrate: 256000 //256kbit
+  };
+
+  encoder.configure(config);
+
+  let timestamp = 0;
+  for (let i = 0; i < frame_count; i++) {
+    // one tenth of a seconds per frame
+    let length = config.sampleRate / 10;
+    timestamp += 100_000;
+    let frame = make_audio_frame(timestamp, config.numberOfChannels,
+                                 config.sampleRate, length);
+    encoder.encode(frame);
+  }
+  await encoder.flush();
+  encoder.close();
+  assert_greater_than_equal(outputs.length, frame_count);
+  for (chunk of outputs) {
+    assert_greater_than(chunk.data.byteLength, 0);
+    assert_greater_than(timestamp, chunk.timestamp);
+  }
+}, 'Simple audio encoding');