Adding input and output delay estimation for mac.

Review URL: http://codereview.chromium.org/8234009

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@107642 0039d316-1c4b-4281-b951-d872f2087c98

media/audio/mac/audio_low_latency_input_mac.cc

@@ -33,7 +33,9 @@ AUAudioInputStream::AUAudioInputStream(
     : manager_(manager),
       sink_(NULL),
       audio_unit_(0),
-      started_(false) {
+      input_device_id_(kAudioObjectUnknown),
+      started_(false),
+      hardware_latency_frames_(0) {
   DCHECK(manager_);
 
   // Set up the desired (output) format specified by the client.
@@ -137,25 +139,34 @@ bool AUAudioInputStream::Open() {
 
   // Set the current device of the AudioOuputUnit to default input device.
 
-  AudioDeviceID input_device;
-  UInt32 size = sizeof(input_device);
-
   // First, obtain the current input device selected by the user.
-  result = AudioHardwareGetProperty(kAudioHardwarePropertyDefaultInputDevice,
-                                    &size,
-                                    &input_device);
+  AudioObjectPropertyAddress default_intput_device_address = {
+    kAudioHardwarePropertyDefaultInputDevice,
+    kAudioObjectPropertyScopeGlobal,
+    kAudioObjectPropertyElementMaster
+  };
+  AudioDeviceID input_device = kAudioObjectUnknown;
+  UInt32 size = sizeof(input_device);
+  result = AudioObjectGetPropertyData(kAudioObjectSystemObject,
+                                      &default_intput_device_address,
+                                      0,
+                                      0,
+                                      &size,
+                                      &input_device);
   if (result) {
     HandleError(result);
     return false;
   }
 
+  input_device_id_ = input_device;
+
   // Next, set the audio device to be the Audio Unit's current device.
   // Note that, devices can only be set to the AUHAL after enabling IO.
   result = AudioUnitSetProperty(audio_unit_,
                                 kAudioOutputUnitProperty_CurrentDevice,
                                 kAudioUnitScope_Global,
                                 0,
-                                &input_device,
+                                &input_device_id_,
                                 sizeof(input_device));
   if (result) {
     HandleError(result);
@@ -213,6 +224,10 @@ bool AUAudioInputStream::Open() {
     HandleError(result);
     return false;
   }
+
+  // The hardware latency is fixed and will not change during the call.
+  hardware_latency_frames_ = GetHardwareLatency();
+
   return true;
 }
 
@@ -290,26 +305,32 @@ OSStatus AUAudioInputStream::InputProc(void* user_data,
 
   // Deliver recorded data to the consumer as a callback.
   return audio_input->Provide(number_of_frames,
-                              audio_input->audio_buffer_list());
+                              audio_input->audio_buffer_list(),
+                              time_stamp);
 }
 
 OSStatus AUAudioInputStream::Provide(UInt32 number_of_frames,
-                                     AudioBufferList* io_data) {
+                                     AudioBufferList* io_data,
+                                     const AudioTimeStamp* time_stamp) {
+  // Update the capture latency.
+  double capture_latency_frames = GetCaptureLatency(time_stamp);
+
   AudioBuffer& buffer = io_data->mBuffers[0];
   uint8* audio_data = reinterpret_cast<uint8*>(buffer.mData);
+  uint32 capture_delay_bytes = static_cast<uint32>
+      ((capture_latency_frames + 0.5) * format_.mBytesPerFrame);
   DCHECK(audio_data);
   if (!audio_data)
     return kAudioUnitErr_InvalidElement;
 
-  // TODO(henrika): improve delay estimation. Using buffer size for now.
-  sink_->OnData(this, audio_data, buffer.mDataByteSize, buffer.mDataByteSize);
+  sink_->OnData(this, audio_data, buffer.mDataByteSize, capture_delay_bytes);
 
   return noErr;
 }
 
 double AUAudioInputStream::HardwareSampleRate() {
   // Determine the default input device's sample-rate.
-  AudioDeviceID device_id = kAudioDeviceUnknown;
+  AudioDeviceID device_id = kAudioObjectUnknown;
   UInt32 info_size = sizeof(device_id);
 
   AudioObjectPropertyAddress default_input_device_address = {
@@ -348,6 +369,89 @@ double AUAudioInputStream::HardwareSampleRate() {
   return nominal_sample_rate;
 }
 
+double AUAudioInputStream::GetHardwareLatency() {
+  if (!audio_unit_ || input_device_id_ == kAudioObjectUnknown) {
+    DLOG(WARNING) << "Audio unit object is NULL or device ID is unknown";
+    return 0.0;
+  }
+
+  // Get audio unit latency.
+  Float64 audio_unit_latency_sec = 0.0;
+  UInt32 size = sizeof(audio_unit_latency_sec);
+  OSStatus result = AudioUnitGetProperty(audio_unit_,
+                                         kAudioUnitProperty_Latency,
+                                         kAudioUnitScope_Global,
+                                         0,
+                                         &audio_unit_latency_sec,
+                                         &size);
+  DLOG_IF(WARNING, result != noErr) << "Could not get audio unit latency.";
+
+  // Get input audio device latency.
+  AudioObjectPropertyAddress property_address = {
+    kAudioDevicePropertyLatency,
+    kAudioDevicePropertyScopeInput,
+    kAudioObjectPropertyElementMaster
+  };
+  UInt32 device_latency_frames = 0;
+  size = sizeof(device_latency_frames);
+  result = AudioObjectGetPropertyData(input_device_id_,
+                                      &property_address,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      &device_latency_frames);
+  DLOG_IF(WARNING, result != noErr) << "Could not get audio device latency.";
+
+  // Get the stream latency.
+  property_address.mSelector = kAudioDevicePropertyStreams;
+  UInt32 stream_latency_frames = 0;
+  size = 0;
+  result = AudioObjectGetPropertyDataSize(input_device_id_,
+                                          &property_address,
+                                          0,
+                                          NULL,
+                                          &size);
+  if (!result) {
+    scoped_ptr_malloc<AudioStreamID>
+        streams(reinterpret_cast<AudioStreamID*>(malloc(size)));
+    AudioStreamID* stream_ids = streams.get();
+    result = AudioObjectGetPropertyData(input_device_id_,
+                                        &property_address,
+                                        0,
+                                        NULL,
+                                        &size,
+                                        stream_ids);
+    if (!result) {
+      property_address.mSelector = kAudioStreamPropertyLatency;
+      result = AudioObjectGetPropertyData(stream_ids[0],
+                                          &property_address,
+                                          0,
+                                          NULL,
+                                          &size,
+                                          &stream_latency_frames);
+    }
+  }
+  DLOG_IF(WARNING, result != noErr) << "Could not get audio stream latency.";
+
+  return static_cast<double>((audio_unit_latency_sec *
+      format_.mSampleRate) + device_latency_frames + stream_latency_frames);
+}
+
+double AUAudioInputStream::GetCaptureLatency(
+    const AudioTimeStamp* input_time_stamp) {
+  // Get the delay between between the actual recording instant and the time
+  // when the data packet is provided as a callback.
+  UInt64 capture_time_ns = AudioConvertHostTimeToNanos(
+      input_time_stamp->mHostTime);
+  UInt64 now_ns = AudioConvertHostTimeToNanos(AudioGetCurrentHostTime());
+  double delay_frames = static_cast<double>
+      (1e-9 * (now_ns - capture_time_ns) * format_.mSampleRate);
+
+  // Total latency is composed by the dynamic latency and the fixed
+  // hardware latency.
+  return (delay_frames + hardware_latency_frames_);
+}
+
 void AUAudioInputStream::HandleError(OSStatus err) {
   NOTREACHED() << "error code: " << err;
   if (sink_)

media/audio/mac/audio_low_latency_input_mac.h

@@ -27,6 +27,11 @@
 //   input device and then use the same rate when creating this object.
 //   Use AUAudioInputStream::HardwareSampleRate() to retrieve the sample rate.
 // - Calling Close() also leads to self destruction.
+// - The latency consists of two parts:
+//   1) Hardware latency, which includes Audio Unit latency, audio device
+//      latency and audio stream latency;
+//   2) The delay between the actual recording instant and the time when the
+//      data packet is provided as a callback.
 //
 #ifndef MEDIA_AUDIO_MAC_AUDIO_LOW_LATENCY_INPUT_MAC_H_
 #define MEDIA_AUDIO_MAC_AUDIO_LOW_LATENCY_INPUT_MAC_H_
@@ -73,7 +78,15 @@ class AUAudioInputStream : public AudioInputStream {
                             AudioBufferList* io_data);
 
   // Pushes recorded data to consumer of the input audio stream.
-  OSStatus Provide(UInt32 number_of_frames, AudioBufferList* io_data);
+  OSStatus Provide(UInt32 number_of_frames, AudioBufferList* io_data,
+                   const AudioTimeStamp* time_stamp);
+
+  // Gets the fixed capture hardware latency and store it during initialization.
+  // Returns 0 if not available.
+  double GetHardwareLatency();
+
+  // Gets the current capture delay value.
+  double GetCaptureLatency(const AudioTimeStamp* input_time_stamp);
 
   // Issues the OnError() callback to the |sink_|.
   void HandleError(OSStatus err);
@@ -96,6 +109,9 @@ class AUAudioInputStream : public AudioInputStream {
   // The AUHAL also enables selection of non default devices.
   AudioUnit audio_unit_;
 
+  // The UID refers to the current input audio device.
+  AudioDeviceID input_device_id_;
+
   // Provides a mechanism for encapsulating one or more buffers of audio data.
   AudioBufferList audio_buffer_list_;
 
@@ -106,6 +122,9 @@ class AUAudioInputStream : public AudioInputStream {
   // True after successfull Start(), false after successful Stop().
   bool started_;
 
+  // Fixed capture hardware latency in frames.
+  double hardware_latency_frames_;
+
   DISALLOW_COPY_AND_ASSIGN(AUAudioInputStream);
 };
 

media/audio/mac/audio_low_latency_input_mac_unittest.cc

@@ -13,8 +13,10 @@
 #include "testing/gmock/include/gmock/gmock.h"
 #include "testing/gtest/include/gtest/gtest.h"
 
+using ::testing::_;
 using ::testing::AnyNumber;
 using ::testing::Between;
+using ::testing::Ge;
 using ::testing::NotNull;
 
 class MockAudioInputCallback : public AudioInputStream::AudioInputCallback {
@@ -215,7 +217,8 @@ TEST(MacAudioInputTest, AUAudioInputStreamVerifyMonoRecording) {
   // startup sequence takes some time, it is reasonable to expect 5-10
   // callbacks in this time period. All should contain valid packets of
   // the same size.
-  EXPECT_CALL(sink, OnData(ais, NotNull(), bytes_per_packet, bytes_per_packet))
+  EXPECT_CALL(sink, OnData(ais, NotNull(), bytes_per_packet,
+                           Ge(bytes_per_packet)))
       .Times(Between(5, 10));
 
   ais->Start(&sink);
@@ -248,7 +251,8 @@ TEST(MacAudioInputTest, AUAudioInputStreamVerifyStereoRecording) {
   // startup sequence takes some time, it is reasonable to expect 5-10
   // callbacks in this time period. All should contain valid packets of
   // the same size.
-  EXPECT_CALL(sink, OnData(ais, NotNull(), bytes_per_packet, bytes_per_packet))
+  EXPECT_CALL(sink, OnData(ais, NotNull(), bytes_per_packet,
+                           Ge(bytes_per_packet)))
       .Times(Between(5, 10));
 
   ais->Start(&sink);

media/audio/mac/audio_low_latency_output_mac.cc

@@ -47,7 +47,9 @@ AUAudioOutputStream::AUAudioOutputStream(
     : manager_(manager),
       source_(NULL),
       output_unit_(0),
-      volume_(1) {
+      output_device_id_(kAudioObjectUnknown),
+      volume_(1),
+      hardware_latency_frames_(0) {
   // We must have a manager.
   DCHECK(manager_);
   // A frame is one sample across all channels. In interleaved audio the per
@@ -72,6 +74,23 @@ AUAudioOutputStream::~AUAudioOutputStream() {
 }
 
 bool AUAudioOutputStream::Open() {
+  // Obtain the current input device selected by the user.
+  UInt32 size = sizeof(output_device_id_);
+  AudioObjectPropertyAddress default_output_device_address = {
+    kAudioHardwarePropertyDefaultOutputDevice,
+    kAudioObjectPropertyScopeGlobal,
+    kAudioObjectPropertyElementMaster
+  };
+  OSStatus result = AudioObjectGetPropertyData(kAudioObjectSystemObject,
+                                               &default_output_device_address,
+                                               0,
+                                               0,
+                                               &size,
+                                               &output_device_id_);
+  DCHECK_EQ(result, 0);
+  if (result)
+    return false;
+
   // Open and initialize the DefaultOutputUnit.
   Component comp;
   ComponentDescription desc;
@@ -84,7 +103,7 @@ bool AUAudioOutputStream::Open() {
   comp = FindNextComponent(0, &desc);
   DCHECK(comp);
 
-  OSStatus result = OpenAComponent(comp, &output_unit_);
+  result = OpenAComponent(comp, &output_unit_);
   DCHECK_EQ(result, 0);
   if (result)
     return false;
@@ -95,6 +114,8 @@ bool AUAudioOutputStream::Open() {
   if (result)
     return false;
 
+  hardware_latency_frames_ = GetHardwareLatency();
+
   return Configure();
 }
 
@@ -185,11 +206,18 @@ void AUAudioOutputStream::GetVolume(double* volume) {
 // Note to future hackers of this function: Do not add locks here because this
 // is running on a real-time thread (for low-latency).
 OSStatus AUAudioOutputStream::Render(UInt32 number_of_frames,
-                                     AudioBufferList* io_data) {
+                                     AudioBufferList* io_data,
+                                     const AudioTimeStamp* output_time_stamp) {
+  // Update the playout latency.
+  double playout_latency_frames = GetPlayoutLatency(output_time_stamp);
+
   AudioBuffer& buffer = io_data->mBuffers[0];
   uint8* audio_data = reinterpret_cast<uint8*>(buffer.mData);
+  uint32 hardware_pending_bytes = static_cast<uint32>
+      ((playout_latency_frames + 0.5) * format_.mBytesPerFrame);
   uint32 filled = source_->OnMoreData(
-      this, audio_data, buffer.mDataByteSize, AudioBuffersState(0, 0));
+      this, audio_data, buffer.mDataByteSize,
+      AudioBuffersState(0, hardware_pending_bytes));
 
   // Handle channel order for 5.1 audio.
   if (format_.mChannelsPerFrame == 6) {
@@ -208,7 +236,7 @@ OSStatus AUAudioOutputStream::Render(UInt32 number_of_frames,
 // DefaultOutputUnit callback
 OSStatus AUAudioOutputStream::InputProc(void* user_data,
                                         AudioUnitRenderActionFlags*,
-                                        const AudioTimeStamp*,
+                                        const AudioTimeStamp* output_time_stamp,
                                         UInt32,
                                         UInt32 number_of_frames,
                                         AudioBufferList* io_data) {
@@ -218,12 +246,12 @@ OSStatus AUAudioOutputStream::InputProc(void* user_data,
   if (!audio_output)
     return -1;
 
-  return audio_output->Render(number_of_frames, io_data);
+  return audio_output->Render(number_of_frames, io_data, output_time_stamp);
 }
 
 double AUAudioOutputStream::HardwareSampleRate() {
   // Determine the default output device's sample-rate.
-  AudioDeviceID device_id = kAudioDeviceUnknown;
+  AudioDeviceID device_id = kAudioObjectUnknown;
   UInt32 info_size = sizeof(device_id);
 
   AudioObjectPropertyAddress default_output_device_address = {
@@ -261,3 +289,83 @@ double AUAudioOutputStream::HardwareSampleRate() {
 
   return nominal_sample_rate;
 }
+
+double AUAudioOutputStream::GetHardwareLatency() {
+  if (!output_unit_ || output_device_id_ == kAudioObjectUnknown) {
+    DLOG(WARNING) << "Audio unit object is NULL or device ID is unknown";
+    return 0.0;
+  }
+
+  // Get audio unit latency.
+  Float64 audio_unit_latency_sec = 0.0;
+  UInt32 size = sizeof(audio_unit_latency_sec);
+  OSStatus result = AudioUnitGetProperty(output_unit_,
+                                         kAudioUnitProperty_Latency,
+                                         kAudioUnitScope_Global,
+                                         0,
+                                         &audio_unit_latency_sec,
+                                         &size);
+  DLOG_IF(WARNING, result != noErr) << "Could not get audio unit latency.";
+
+  // Get output audio device latency.
+  AudioObjectPropertyAddress property_address = {
+    kAudioDevicePropertyLatency,
+    kAudioDevicePropertyScopeOutput,
+    kAudioObjectPropertyElementMaster
+  };
+  UInt32 device_latency_frames = 0;
+  size = sizeof(device_latency_frames);
+  result = AudioObjectGetPropertyData(output_device_id_,
+                                      &property_address,
+                                      0,
+                                      NULL,
+                                      &size,
+                                      &device_latency_frames);
+  DLOG_IF(WARNING, result != noErr) << "Could not get audio device latency.";
+
+  // Get the stream latency.
+  property_address.mSelector = kAudioDevicePropertyStreams;
+  UInt32 stream_latency_frames = 0;
+  result = AudioObjectGetPropertyDataSize(output_device_id_,
+                                          &property_address,
+                                          0,
+                                          NULL,
+                                          &size);
+  if (!result) {
+    scoped_ptr_malloc<AudioStreamID>
+        streams(reinterpret_cast<AudioStreamID*>(malloc(size)));
+    AudioStreamID* stream_ids = streams.get();
+    result = AudioObjectGetPropertyData(output_device_id_,
+                                        &property_address,
+                                        0,
+                                        NULL,
+                                        &size,
+                                        stream_ids);
+    if (!result) {
+      property_address.mSelector = kAudioStreamPropertyLatency;
+      result = AudioObjectGetPropertyData(stream_ids[0],
+                                          &property_address,
+                                          0,
+                                          NULL,
+                                          &size,
+                                          &stream_latency_frames);
+    }
+  }
+  DLOG_IF(WARNING, result != noErr) << "Could not get audio stream latency.";
+
+  return static_cast<double>((audio_unit_latency_sec *
+      format_.mSampleRate) + device_latency_frames + stream_latency_frames);
+}
+
+double AUAudioOutputStream::GetPlayoutLatency(
+    const AudioTimeStamp* output_time_stamp) {
+  // Get the delay between the moment getting the callback and the scheduled
+  // time stamp that tells when the data is going to be played out.
+  UInt64 output_time_ns = AudioConvertHostTimeToNanos(
+      output_time_stamp->mHostTime);
+  UInt64 now_ns = AudioConvertHostTimeToNanos(AudioGetCurrentHostTime());
+  double delay_frames = static_cast<double>
+      (1e-9 * (output_time_ns - now_ns) * format_.mSampleRate);
+
+  return (delay_frames + hardware_latency_frames_);
+}

media/audio/mac/audio_low_latency_output_mac.h

 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
-
+//
+// Implementation notes:
+//
+// - It is recommended to first acquire the native sample rate of the default
+//   output device and then use the same rate when creating this object.
+//   Use AUAudioOutputStream::HardwareSampleRate() to retrieve the sample rate.
+// - Calling Close() also leads to self destruction.
+// - The latency consists of two parts:
+//   1) Hardware latency, which includes Audio Unit latency, audio device
+//      latency and audio stream latency;
+//   2) The delay between the moment getting the callback and the scheduled time
+//      stamp that tells when the data is going to be played out.
+//
 #ifndef MEDIA_AUDIO_MAC_AUDIO_LOW_LATENCY_OUTPUT_MAC_H_
 #define MEDIA_AUDIO_MAC_AUDIO_LOW_LATENCY_OUTPUT_MAC_H_
 
 #include <AudioUnit/AudioUnit.h>
 
+#include "base/memory/scoped_ptr.h"
 #include "media/audio/audio_io.h"
 #include "media/audio/audio_parameters.h"
 
@@ -44,11 +57,19 @@ class AUAudioOutputStream : public AudioOutputStream {
                             UInt32 number_of_frames,
                             AudioBufferList* io_data);
 
-  OSStatus Render(UInt32 number_of_frames, AudioBufferList* io_data);
+  OSStatus Render(UInt32 number_of_frames, AudioBufferList* io_data,
+                  const AudioTimeStamp* output_time_stamp);
 
   // Sets up the stream format for the default output Audio Unit.
   bool Configure();
 
+  // Gets the fixed playout device hardware latency and stores it. Returns 0
+  // if not available.
+  double GetHardwareLatency();
+
+  // Gets the current playout latency value.
+  double GetPlayoutLatency(const AudioTimeStamp* output_time_stamp);
+
   // Our creator, the audio manager needs to be notified when we close.
   AudioManagerMac* manager_;
 
@@ -63,9 +84,15 @@ class AUAudioOutputStream : public AudioOutputStream {
   // The default output Audio Unit which talks to the audio hardware.
   AudioUnit output_unit_;
 
+  // The UID refers to the current output audio device.
+  AudioDeviceID output_device_id_;
+
   // Volume level from 0 to 1.
   float volume_;
 
+  // Fixed playout hardware latency in frames.
+  double hardware_latency_frames_;
+
   DISALLOW_COPY_AND_ASSIGN(AUAudioOutputStream);
 };