[Chromecast] Add loopback input for AEC

Add i2s loopback in order to enable the reference channel for AEC.

Bug: http://b/38428792
Test: built cast_shell_internal_apk, ran assistant with AEC
Change-Id: Ie8c1ecf09bc31050e3cfe61eddf69dc21aba1fce
Reviewed-on: https://chromium-review.googlesource.com/566063
Commit-Queue: Nat Jeffries <njeff@google.com>
Reviewed-by: Luke Halliwell <halliwell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#488801}

chromecast/BUILD.gn

@@ -485,6 +485,7 @@ buildflag_header("chromecast_features") {
   header = "chromecast_features.h"
   flags = [
     "ENABLE_ASSISTANT=$enable_assistant",
+    "ENABLE_ATHINGS_LOOPBACK=$enable_athings_loopback",
     "IS_ANDROID_THINGS=$is_android_things",
     "IS_CAST_AUDIO_ONLY=$is_cast_audio_only",
     "IS_CAST_USING_CMA_BACKEND=$is_cast_using_cma_backend",

chromecast/base/chromecast_switches.cc

@@ -109,6 +109,12 @@ const char kCastInitialScreenWidth[] = "cast-initial-screen-width";
 const char kCastInitialScreenHeight[] = "cast-initial-screen-height";
 const char kUseDoubleBuffering[] = "use-double-buffering";
 
+// Used to pass configuration for the I2S input to enable loopback for AEC.
+const char kLoopbackI2sBits[] = "loopback-i2s-bits";
+const char kLoopbackI2sChannels[] = "loopback-i2s-channels";
+const char kLoopbackI2sNumber[] = "loopback-i2s-number";
+const char kLoopbackI2sRate[] = "loopback-i2s-rate";
+
 // When present, desktop cast_shell will create 1080p window (provided display
 // resolution is high enough).  Otherwise, cast_shell defaults to 720p.
 const char kDesktopWindow1080p[] = "desktop-window-1080p";

chromecast/base/chromecast_switches.h

@@ -60,6 +60,12 @@ extern const char kCastInitialScreenWidth[];
 extern const char kCastInitialScreenHeight[];
 extern const char kUseDoubleBuffering[];
 
+// I2S loopback configuration switches
+extern const char kLoopbackI2sBits[];
+extern const char kLoopbackI2sChannels[];
+extern const char kLoopbackI2sNumber[];
+extern const char kLoopbackI2sRate[];
+
 // Graphics switches
 extern const char kDesktopWindow1080p[];
 

chromecast/chromecast.gni

@@ -49,6 +49,12 @@ declare_args() {
   is_android_things = false
 }
 
+declare_args() {
+  # Currently android things libraries live in internal.  TODO(njeff): change
+  # this when Android Things API is moved to public
+  enable_athings_loopback = is_android_things && chromecast_branding != "public"
+}
+
 declare_args() {
   # Use Playready CDMs for internal non-desktop builds.
   use_playready = !is_cast_desktop_build && chromecast_branding != "public"

chromecast/media/cma/backend/android/BUILD.gn

@@ -4,6 +4,7 @@
 
 import("//build/config/android/config.gni")
 import("//build/config/android/rules.gni")
+import("//chromecast/chromecast.gni")
 
 source_set("cast_media_android") {
   sources = [
@@ -34,6 +35,14 @@ source_set("cast_media_android") {
     "//chromecast/public/media",
     "//media",
   ]
+
+  if (enable_athings_loopback) {
+    deps += [ "//chromecast/internal/android/prebuilt/things:support_lib" ]
+    sources += [
+      "loopback_audio_manager.cc",
+      "loopback_audio_manager.h",
+    ]
+  }
 }
 
 generate_jni("audio_track_jni_headers") {

chromecast/media/cma/backend/android/DEPS

 include_rules = [
   "+jni",
   "+media/filters",
+  "+chromecast/internal/android/prebuilt/things",
 ]

chromecast/media/cma/backend/android/cast_media_android.cc

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+
 #include "base/at_exit.h"
 #include "base/logging.h"
+#include "chromecast/chromecast_features.h"
+#include "chromecast/media/cma/backend/android/loopback_audio_manager.h"
 #include "chromecast/media/cma/backend/android/media_pipeline_backend_android.h"
 #include "chromecast/public/cast_media_shlib.h"
 #include "chromecast/public/graphics_types.h"
@@ -90,16 +93,16 @@ bool CastMediaShlib::SupportsMediaClockRateChange() {
   return false;
 }
 
-#if 0  // disable for now, since libassistant doesn't handle a stub well.
+#if BUILDFLAG(ENABLE_ATHINGS_LOOPBACK)
 void CastMediaShlib::AddLoopbackAudioObserver(LoopbackAudioObserver* observer) {
   LOG(INFO) << __func__ << ":";
-  // TODO(ckuiper): Hook-up if applicable.
+  LoopbackAudioManager::Get()->AddLoopbackAudioObserver(observer);
 }
 
 void CastMediaShlib::RemoveLoopbackAudioObserver(
     LoopbackAudioObserver* observer) {
   LOG(INFO) << __func__ << ":";
-  // TODO(ckuiper): Hook-up if applicable.
+  LoopbackAudioManager::Get()->RemoveLoopbackAudioObserver(observer);
 }
 #endif
 

chromecast/media/cma/backend/android/loopback_audio_manager.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromecast/media/cma/backend/android/loopback_audio_manager.h"
+
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <string>
+
+#include "base/bind.h"
+#include "base/command_line.h"
+#include "base/lazy_instance.h"
+#include "base/logging.h"
+#include "chromecast/base/chromecast_switches.h"
+#include "chromecast/base/task_runner_impl.h"
+#include "chromecast/internal/android/prebuilt/things/include/pio/i2s_device.h"
+#include "chromecast/internal/android/prebuilt/things/include/pio/peripheral_manager_client.h"
+#include "chromecast/public/cast_media_shlib.h"
+#include "chromecast/public/media/decoder_config.h"
+
+#define RUN_ON_FEEDER_THREAD(method, ...)                        \
+  if (!feeder_thread_.task_runner()->BelongsToCurrentThread()) { \
+    POST_TASK_TO_FEEDER_THREAD(method, ##__VA_ARGS__);           \
+    return;                                                      \
+  }
+
+#define POST_TASK_TO_FEEDER_THREAD(method, ...)                \
+  feeder_thread_.task_runner()->PostTask(                      \
+      FROM_HERE, base::BindOnce(&LoopbackAudioManager::method, \
+                                base::Unretained(this), ##__VA_ARGS__));
+
+namespace chromecast {
+namespace media {
+
+namespace {
+
+const int kMaxI2sNameLen = 32;
+const int kBufferLenMs = 20;
+const int kMsPerSecond 1000;
+const int64_t kNsecPerSecond = 1000000000LL;
+const int64_t kTimestampUpdatePeriodNsec = 10 * kNsecPerSecond;
+const int64_t kInvalidTimestamp = std::numeric_limits<int64_t>::min();
+
+class LoopbackAudioManagerInstance : public LoopbackAudioManager {
+ public:
+  LoopbackAudioManagerInstance() {}
+  ~LoopbackAudioManagerInstance() override {}
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(LoopbackAudioManagerInstance);
+};
+
+base::LazyInstance<LoopbackAudioManagerInstance>::DestructorAtExit
+    loopback_audio_manager_instance = LAZY_INSTANCE_INITIALIZER;
+
+}  // namespace
+
+LoopbackAudioManager::LoopbackAudioManager()
+    : feeder_thread_("Android_Loopback"),
+      last_timestamp_nsec_(kInvalidTimestamp),
+      last_frame_position_(0),
+      frame_count_(0),
+      loopback_running(false) {
+  base::Thread::Options options;
+  options.priority = base::ThreadPriority::REALTIME_AUDIO;
+  CHECK(feeder_thread_.StartWithOptions(options));
+}
+
+LoopbackAudioManager::~LoopbackAudioManager() {
+  DCHECK(loopback_observers_.empty());
+  feeder_thread_.Stop();
+
+  // thread_.Stop() makes sure the thread is stopped before return.
+  // It's okay to clean up after feeder_thread_ is stopped.
+  StopLoopback();
+}
+
+// static
+LoopbackAudioManager* LoopbackAudioManager::Get() {
+  return loopback_audio_manager_instance.Pointer();
+}
+
+void LoopbackAudioManager::CalibrateTimestamp(int64_t frame_position) {
+  // Determine if a new calibration timestamp is needed.
+  struct timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  int64_t time_sec = static_cast<int64_t>(ts.tv_sec);
+  int64_t time_nsec = static_cast<int64_t>(ts.tv_nsec);
+  time_nsec += time_sec * kNsecPerSecond;
+  if (last_timestamp_nsec_ != kInvalidTimestamp &&
+      time_nsec - last_timestamp_nsec_ < kTimestampUpdatePeriodNsec) {
+    return;
+  }
+
+  // Validate that timestamp is close to interpolated estimate.
+  int64_t old_last_position = last_frame_position_;
+  int64_t old_last_timestamp = last_timestamp_nsec_;
+  int success;
+  DCHECK_EQ(0, AI2sDevice_getInputTimestamp(i2s_, &last_frame_position_,
+                                            &last_timestamp_nsec_, &success));
+  // If the call fails, the values are not updated.
+  if (!success) {
+    return;
+  }
+  int64_t delta_frames = last_frame_position_ - old_last_position;
+  int64_t delta_nsecs = kNsecPerSecond * delta_frames / i2s_rate_;
+  int64_t expected_timestamp = old_last_timestamp + delta_nsecs;
+  int64_t ts_diff_micros = (last_timestamp_nsec_ - expected_timestamp) / 1000;
+  if (ts_diff_micros > 1000) {
+    LOG(WARNING) << "Updated timestamp, interpolated timestamp drifted by "
+                 << ts_diff_micros << " microseconds";
+  }
+}
+
+int64_t LoopbackAudioManager::GetInterpolatedTimestamp(int64_t frame_position) {
+  CalibrateTimestamp(frame_position);
+  int64_t delta_frames =
+      frame_position - last_frame_position_ -
+      audio_buffer_size_ / (bytes_per_sample_ * i2s_channels_);
+  int64_t delta_nsecs = kNsecPerSecond * delta_frames / i2s_rate_;
+  return last_timestamp_nsec_ + delta_nsecs;
+}
+
+void LoopbackAudioManager::GetI2sFlags(char* i2s_name,
+                                       AI2sEncoding* i2s_encoding) {
+  int i2s_number = GetSwitchValueInt(switches::kLoopbackI2sNumber, -1);
+  LOG_IF(DFATAL, i2s_number == -1)
+      << "Flag --" << switches::kLoopbackI2sNumber << " is required.";
+
+  sprintf(i2s_name, "I2S%d", i2s_number);
+  i2s_rate_ = GetSwitchValueNonNegativeInt(switches::kLoopbackI2sRate, 0);
+  LOG_IF(DFATAL, !i2s_rate_)
+      << "Flag --" << switches::kLoopbackI2sRate << " is required.";
+
+  int i2s_bits = GetSwitchValueNonNegativeInt(switches::kLoopbackI2sBits, 0);
+  LOG_IF(DFATAL, !i2s_bits)
+      << "Flag --" << switches::kLoopbackI2sBits << " is required.";
+  switch (i2s_bits) {
+    case 16:
+      *i2s_encoding = AI2S_ENCODING_PCM_16_BIT;
+      cast_audio_format_ = kSampleFormatS16;
+      bytes_per_sample_ = 2;
+      break;
+    case 24:
+      *i2s_encoding = AI2S_ENCODING_PCM_24_BIT;
+      cast_audio_format_ = kSampleFormatS24;
+      bytes_per_sample_ = 4;  // 24-bits algined to 32 bits
+      break;
+    case 32:
+      *i2s_encoding = AI2S_ENCODING_PCM_32_BIT;
+      cast_audio_format_ = kSampleFormatS32;
+      bytes_per_sample_ = 4;
+      break;
+    default:
+      LOG(FATAL)
+          << "Invalid number of bits specified.  Must be one of {16, 24, 32}";
+  }
+
+  i2s_channels_ =
+      GetSwitchValueNonNegativeInt(switches::kLoopbackI2sChannels, 0);
+  LOG_IF(DFATAL, !i2s_channels_)
+      << "Flag --" << switches::kLoopbackI2sChannels << " is required.";
+
+  // 20ms of audio
+  audio_buffer_size_ =
+      i2s_rate_ * bytes_per_sample_ * i2s_channels_ * kBufferLenMs / kMsPerSecond;
+}
+
+void LoopbackAudioManager::StartLoopback() {
+  DCHECK(feeder_thread_.task_runner()->BelongsToCurrentThread());
+  DCHECK(!loopback_running_);
+  last_timestamp_nsec_ = kInvalidTimestamp;
+
+  // Open I2S device.
+  APeripheralManagerClient* client = APeripheralManagerClient_new();
+  DCHECK(client);
+
+  char i2s_name[kMaxI2sNameLen];
+  AI2sEncoding i2s_encoding;
+  GetI2sFlags(i2s_name, &i2s_encoding);
+
+  int err = APeripheralManagerClient_openI2sDevice(
+      client, i2s_name, i2s_encoding, i2s_channels_, i2s_rate_,
+      AI2S_FLAG_DIRECTION_IN, &i2s_);
+  DCHECK_EQ(err, 0);
+
+  // Maintain sample count for interpolation.
+  frame_count_ = 0;
+  loopback_running_ = true;
+  RunLoopback();
+}
+
+void LoopbackAudioManager::StopLoopback() {
+  DCHECK(loopback_running_);
+  loopback_running_ = false;
+  AI2sDevice_delete(i2s_);
+}
+
+void LoopbackAudioManager::RunLoopback() {
+  DCHECK(feeder_thread_.task_runner()->BelongsToCurrentThread());
+
+  if (!loopback_running_) {
+    return;
+  }
+
+  // Read bytes from I2S device.
+  int bytes_read;
+  uint8_t data[audio_buffer_size_];
+  AI2sDevice_read(i2s_, data, 0, audio_buffer_size_, &bytes_read);
+  DCHECK_EQ(audio_buffer_size_, bytes_read);
+  frame_count_ += bytes_read / (bytes_per_sample_ * i2s_channels_);
+
+  // Get high-resolution timestamp.
+  int64_t timestamp_ns = GetInterpolatedTimestamp(frame_count_);
+
+  // Post data and timestamp.
+  for (auto* observer : loopback_observers_) {
+    observer->OnLoopbackAudio(timestamp_ns / 1000, cast_audio_format_,
+                              i2s_rate_, i2s_channels_, data,
+                              audio_buffer_size_);
+  }
+
+  POST_TASK_TO_FEEDER_THREAD(RunLoopback);
+}
+
+void LoopbackAudioManager::AddLoopbackAudioObserver(
+    CastMediaShlib::LoopbackAudioObserver* observer) {
+  DCHECK(observer);
+  RUN_ON_FEEDER_THREAD(AddLoopbackAudioObserver, observer);
+
+  DCHECK(std::find(loopback_observers_.begin(), loopback_observers_.end(),
+                   observer) == loopback_observers_.end());
+  loopback_observers_.push_back(observer);
+  if (loopback_observers_.size() == 1) {
+    StartLoopback();
+  }
+}
+
+void LoopbackAudioManager::RemoveLoopbackAudioObserver(
+    CastMediaShlib::LoopbackAudioObserver* observer) {
+  DCHECK(observer);
+  RUN_ON_FEEDER_THREAD(RemoveLoopbackAudioObserver, observer);
+
+  DCHECK(std::find(loopback_observers_.begin(), loopback_observers_.end(),
+                   observer) != loopback_observers_.end());
+  loopback_observers_.erase(std::remove(loopback_observers_.begin(),
+                                        loopback_observers_.end(), observer),
+                            loopback_observers_.end());
+  observer->OnRemoved();
+  if (loopback_observers_.empty()) {
+    StopLoopback();
+  }
+}
+
+}  // namespace media
+}  // namespace chromecast

chromecast/media/cma/backend/android/loopback_audio_manager.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMECAST_MEDIA_CMA_BACKEND_ANDROID_LOOPBACK_AUDIO_MANAGER_H_
+#define CHROMECAST_MEDIA_CMA_BACKEND_ANDROID_LOOPBACK_AUDIO_MANAGER_H_
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "base/macros.h"
+#include "base/threading/thread.h"
+#include "chromecast/internal/android/prebuilt/things/include/pio/i2s_device.h"
+#include "chromecast/public/cast_media_shlib.h"
+
+struct AI2sDevice;
+
+namespace chromecast {
+namespace media {
+
+class LoopbackAudioManager {
+ public:
+  // Get singleton instance of Loopback Audio Manager
+  static LoopbackAudioManager* Get();
+
+  // Adds a loopback audio observer.
+  void AddLoopbackAudioObserver(
+      CastMediaShlib::LoopbackAudioObserver* observer);
+
+  // Removes a loopback audio observer.
+  void RemoveLoopbackAudioObserver(
+      CastMediaShlib::LoopbackAudioObserver* observer);
+
+ protected:
+  LoopbackAudioManager();
+  virtual ~LoopbackAudioManager();
+
+ private:
+  void StartLoopback();
+  void StopLoopback();
+  void RunLoopback();
+  void CalibrateTimestamp(int64_t frame_position);
+  int64_t GetInterpolatedTimestamp(int64_t frame_position);
+  void GetI2sFlags(char* i2s_name, AI2sEncoding* i2s_encoding);
+
+  std::vector<CastMediaShlib::LoopbackAudioObserver*> loopback_observers_;
+
+  AI2sDevice* i2s_;
+
+  int64_t last_timestamp_nsec_;
+  int64_t last_frame_position_;
+  int64_t frame_count_;
+
+  // Initialized based on the values of castshell flags
+  int audio_buffer_size_;
+  int bytes_per_sample_;
+  int i2s_rate_;
+  int i2s_channels_;
+  SampleFormat cast_audio_format_;
+
+  // Thread that feeds audio data into the observer from the loopback.  The
+  // calls made on this thread are blocking.
+  base::Thread feeder_thread_;
+  bool loopback_running_;
+
+  DISALLOW_COPY_AND_ASSIGN(LoopbackAudioManager);
+};
+
+}  // namespace media
+}  // namespace chromecast
+
+#endif  // CHROMECAST_MEDIA_CMA_BACKEND_ANDROID_LOOPBACK_AUDIO_MANAGER_H_