Update MediaStreamTrack constraints processing algorithm for audio

This CL introduces two main updates to constraints processing for audio:
1. The SelectSettings() algorithm now considers audio-processing
   properties as belonging to each device instead of being considered
   independent of the device. The reason is that, while the
   audio-processing module can be applied to any source, once a source
   has processing applied, all tracks for that source must use the same
   audio-processing settings.
2. The getUserMedia() implementation leverages (1) and makes sure that
   candidate settings for audio sources already in use are limited to
   their current configuration.

Bug: 802198
Change-Id: I3554206e852ff41116a1fe02be45a7a8dff4760d
Reviewed-on: https://chromium-review.googlesource.com/881183Reviewed-by: Henrik Boström <hbos@chromium.org>
Commit-Queue: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#533288}

content/browser/webrtc/webrtc_getusermedia_browsertest.cc

@@ -811,4 +811,12 @@ IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest,
   ExecuteJavascriptAndWaitForOk("getUserMediaAfterStopCanvasCapture()");
 }
 
+IN_PROC_BROWSER_TEST_F(WebRtcGetUserMediaBrowserTest,
+                       GetUserMediaEchoCancellationOnAndOff) {
+  ASSERT_TRUE(embedded_test_server()->Start());
+  GURL url(embedded_test_server()->GetURL("/media/getusermedia.html"));
+  NavigateToURL(shell(), url);
+  ExecuteJavascriptAndWaitForOk("getUserMediaEchoCancellationOnAndOff()");
+}
+
 }  // namespace content

content/renderer/media/media_stream_constraints_util_audio.h

@@ -20,8 +20,63 @@ namespace content {
 
 class MediaStreamAudioSource;
 
+// This class represents the capability of an audio-capture device.
+// It may represent three different things:
+// 1. An audio-capture device that is currently in use.
+// 2. An audio-capture device that is currently not in use, but whose ID and
+//    parameters are known (suitable for device capture, where device IDs are
+//    always known).
+// 3. A "device" whose ID is not known (suitable for content capture, where
+//    it is not possible to have a list of known valid device IDs).
+// In cases (1) and (2), the known device ID introduces a restriction on the
+// acceptable values for the deviceId constraint, while in case (3) no such
+// restriction is imposed and any requested deviceID value will be acceptable
+// while processing constraints.
+class CONTENT_EXPORT AudioDeviceCaptureCapability {
+ public:
+  // This creates an AudioDeviceCaptureCapability that admits all possible
+  // device names and settings. This is intended to be used as the single
+  // capability for getUserMedia() with content capture, where the set of valid
+  // device IDs is infinite.
+  AudioDeviceCaptureCapability();
+
+  // This creates an AudioDeviceCaptureCapability where the device ID is limited
+  // to |device_id| and other settings are limited by the given |parameters|.
+  // |device_id| must not be empty. Intended to be used by getUserMedia() with
+  // device capture for devices that are not currently in use.
+  AudioDeviceCaptureCapability(std::string device_id,
+                               const media::AudioParameters& parameters);
+
+  // This creates an AudioDeviceCaptureCapability where the device ID and other
+  // settings are restricted to the current settings of |source|. Intended to be
+  // used by applyConstraints() for both device and content capture, and by
+  // getUserMedia() with device capture for devices that are currently in use.
+  explicit AudioDeviceCaptureCapability(MediaStreamAudioSource* source);
+
+  // If this capability represents a device currently in use, this method
+  // returns a pointer to the MediaStreamAudioSource object associated with the
+  // device. Otherwise, it returns null.
+  MediaStreamAudioSource* source() const { return source_; }
+
+  // Returns the ID of the device associated with this capability. If empty,
+  // it means that this capability is not associated with a known device and
+  // no restrictions are imposed on the deviceId or other constraints while
+  // processing constraints.
+  const std::string& DeviceID() const;
+
+  // Returns the audio parameters for the device associated with this
+  // capability. If DeviceID() returns an empty string, these parameters contain
+  // default values that work well for content capture.
+  const media::AudioParameters& Parameters() const;
+
+ private:
+  MediaStreamAudioSource* source_ = nullptr;
+  std::string device_id_;
+  media::AudioParameters parameters_;
+};
+
 using AudioDeviceCaptureCapabilities =
-    std::vector<blink::mojom::AudioInputDeviceCapabilitiesPtr>;
+    std::vector<AudioDeviceCaptureCapability>;
 
 // This function implements the SelectSettings algorithm for audio tracks as
 // described in https://w3c.github.io/mediacapture-main/#dfn-selectsettings

content/renderer/media/media_stream_constraints_util_audio_unittest.cc

@@ -63,39 +63,38 @@ class MediaStreamConstraintsUtilAudioTest
  public:
   void SetUp() override {
     ResetFactory();
-    if (!IsDeviceCapture())
-      return;
-
-    blink::mojom::AudioInputDeviceCapabilitiesPtr device =
-        blink::mojom::AudioInputDeviceCapabilities::New();
-    device->device_id = "default_device";
-    device->parameters = media::AudioParameters(
-        media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
-        media::CHANNEL_LAYOUT_STEREO,
-        media::AudioParameters::kAudioCDSampleRate, 16, 1000);
-    capabilities_.push_back(std::move(device));
-
-    device = blink::mojom::AudioInputDeviceCapabilities::New();
-    device->device_id = "hw_echo_canceller_device";
-    device->parameters = media::AudioParameters(
-        media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
-        media::CHANNEL_LAYOUT_STEREO,
-        media::AudioParameters::kAudioCDSampleRate, 24, 1000);
-    device->parameters.set_effects(media::AudioParameters::ECHO_CANCELLER);
-    capabilities_.push_back(std::move(device));
-
-    device = blink::mojom::AudioInputDeviceCapabilities::New();
-    device->device_id = "geometry device";
-    device->parameters = media::AudioParameters(
-        media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
-        media::CHANNEL_LAYOUT_STEREO,
-        media::AudioParameters::kAudioCDSampleRate, 16, 1000);
-    device->parameters.set_mic_positions(kMicPositions);
-    capabilities_.push_back(std::move(device));
-
-    default_device_ = capabilities_[0].get();
-    hw_echo_canceller_device_ = capabilities_[1].get();
-    geometry_device_ = capabilities_[2].get();
+    if (IsDeviceCapture()) {
+      capabilities_.emplace_back(
+          "default_device",
+          media::AudioParameters(media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+                                 media::CHANNEL_LAYOUT_STEREO,
+                                 media::AudioParameters::kAudioCDSampleRate, 16,
+                                 1000));
+
+      media::AudioParameters hw_echo_canceller_parameters(
+          media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+          media::CHANNEL_LAYOUT_STEREO,
+          media::AudioParameters::kAudioCDSampleRate, 24, 1000);
+      hw_echo_canceller_parameters.set_effects(
+          media::AudioParameters::ECHO_CANCELLER);
+      capabilities_.emplace_back("hw_echo_canceller_device",
+                                 hw_echo_canceller_parameters);
+
+      media::AudioParameters geometry_parameters(
+          media::AudioParameters::AUDIO_PCM_LOW_LATENCY,
+          media::CHANNEL_LAYOUT_STEREO,
+          media::AudioParameters::kAudioCDSampleRate, 16, 1000);
+      geometry_parameters.set_mic_positions(kMicPositions);
+      capabilities_.emplace_back("geometry device", geometry_parameters);
+
+      default_device_ = &capabilities_[0];
+      hw_echo_canceller_device_ = &capabilities_[1];
+      geometry_device_ = &capabilities_[2];
+    } else {
+      // For content capture, use a single capability that admits all possible
+      // settings.
+      capabilities_.emplace_back();
+    }
   }
 
  protected:
@@ -125,6 +124,7 @@ class MediaStreamConstraintsUtilAudioTest
       bool disable_local_echo,
       bool render_to_associated_sink) {
     MediaStreamDevice device;
+    device.id = "processed_source";
     device.type = GetMediaStreamType();
     if (render_to_associated_sink)
       device.matched_output_device_id = std::string("some_device_id");
@@ -318,17 +318,16 @@ class MediaStreamConstraintsUtilAudioTest
     }
   }
 
-  void CheckDevice(
-      const blink::mojom::AudioInputDeviceCapabilities& expected_device,
-      const AudioCaptureSettings& result) {
-    EXPECT_EQ(expected_device.device_id, result.device_id());
-    EXPECT_EQ(expected_device.parameters.sample_rate(),
+  void CheckDevice(const AudioDeviceCaptureCapability& expected_device,
+                   const AudioCaptureSettings& result) {
+    EXPECT_EQ(expected_device.DeviceID(), result.device_id());
+    EXPECT_EQ(expected_device.Parameters().sample_rate(),
               result.device_parameters().sample_rate());
-    EXPECT_EQ(expected_device.parameters.bits_per_sample(),
+    EXPECT_EQ(expected_device.Parameters().bits_per_sample(),
               result.device_parameters().bits_per_sample());
-    EXPECT_EQ(expected_device.parameters.channels(),
+    EXPECT_EQ(expected_device.Parameters().channels(),
               result.device_parameters().channels());
-    EXPECT_EQ(expected_device.parameters.effects(),
+    EXPECT_EQ(expected_device.Parameters().effects(),
               result.device_parameters().effects());
   }
 
@@ -355,10 +354,9 @@ class MediaStreamConstraintsUtilAudioTest
 
   MockConstraintFactory constraint_factory_;
   AudioDeviceCaptureCapabilities capabilities_;
-  const blink::mojom::AudioInputDeviceCapabilities* default_device_ = nullptr;
-  const blink::mojom::AudioInputDeviceCapabilities* hw_echo_canceller_device_ =
-      nullptr;
-  const blink::mojom::AudioInputDeviceCapabilities* geometry_device_ = nullptr;
+  const AudioDeviceCaptureCapability* default_device_ = nullptr;
+  const AudioDeviceCaptureCapability* hw_echo_canceller_device_ = nullptr;
+  const AudioDeviceCaptureCapability* geometry_device_ = nullptr;
   const std::vector<media::Point> kMicPositions = {{8, 8, 8}, {4, 4, 4}};
 
  private:
@@ -520,18 +518,18 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, IdealArbitraryDeviceID) {
 TEST_P(MediaStreamConstraintsUtilAudioTest, ExactValidDeviceID) {
   for (const auto& device : capabilities_) {
     constraint_factory_.basic().device_id.SetExact(
-        blink::WebString::FromASCII(device->device_id));
+        blink::WebString::FromASCII(device.DeviceID()));
     auto result = SelectSettings();
     EXPECT_TRUE(result.HasValue());
-    CheckDevice(*device, result);
+    CheckDevice(device, result);
     CheckBoolDefaults(AudioSettingsBoolMembers(),
                       {&AudioProcessingProperties::enable_sw_echo_cancellation},
                       result);
     bool has_hw_echo_cancellation =
-        device->parameters.effects() & media::AudioParameters::ECHO_CANCELLER;
-    EXPECT_EQ(!has_hw_echo_cancellation,
+        device.Parameters().effects() & media::AudioParameters::ECHO_CANCELLER;
+    EXPECT_EQ(IsDeviceCapture() && !has_hw_echo_cancellation,
               result.audio_processing_properties().enable_sw_echo_cancellation);
-    if (&*device == geometry_device_) {
+    if (&device == geometry_device_) {
       EXPECT_EQ(kMicPositions,
                 result.audio_processing_properties().goog_array_geometry);
     } else {
@@ -619,7 +617,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, EchoCancellationWithHw) {
       for (bool value : kBoolValues) {
         ResetFactory();
         constraint_factory_.basic().device_id.SetExact(
-            blink::WebString::FromASCII(hw_echo_canceller_device_->device_id));
+            blink::WebString::FromASCII(hw_echo_canceller_device_->DeviceID()));
         ((constraint_factory_.*accessor)().echo_cancellation.*
          set_function)(value);
         auto result = SelectSettings();
@@ -715,7 +713,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, GoogEchoCancellationWithHw) {
       for (bool value : kBoolValues) {
         ResetFactory();
         constraint_factory_.basic().device_id.SetExact(
-            blink::WebString::FromASCII(hw_echo_canceller_device_->device_id));
+            blink::WebString::FromASCII(hw_echo_canceller_device_->DeviceID()));
         ((constraint_factory_.*accessor)().goog_echo_cancellation.*
          set_function)(value);
         auto result = SelectSettings();
@@ -956,7 +954,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, DeviceGeometry) {
     return;
 
   constraint_factory_.basic().device_id.SetExact(
-      blink::WebString::FromASCII(geometry_device_->device_id));
+      blink::WebString::FromASCII(geometry_device_->DeviceID()));
 
   {
     const blink::WebString kValidGeometry =
@@ -1236,6 +1234,52 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, SourceWithAudioProcessing) {
   }
 }
 
+TEST_P(MediaStreamConstraintsUtilAudioTest, UsedAndUnusedSources) {
+  // The distinction of used and unused sources is relevant only for device
+  // capture.
+  if (!IsDeviceCapture())
+    return;
+
+  AudioProcessingProperties properties;
+  properties.enable_sw_echo_cancellation = true;
+
+  std::unique_ptr<ProcessedLocalAudioSource> processed_source =
+      GetProcessedLocalAudioSource(properties, false /* hotword_enabled */,
+                                   false /* disable_local_echo */,
+                                   false /* render_to_associated_sink */);
+
+  const std::string kUnusedDeviceID = "unused_device";
+  AudioDeviceCaptureCapabilities capabilities;
+  capabilities.emplace_back(processed_source.get());
+  capabilities.emplace_back(kUnusedDeviceID,
+                            media::AudioParameters::UnavailableDeviceParams());
+
+  {
+    constraint_factory_.Reset();
+    constraint_factory_.basic().echo_cancellation.SetExact(false);
+
+    auto result = SelectSettingsAudioCapture(
+        capabilities, constraint_factory_.CreateWebMediaConstraints(),
+        false /* should_disable_hardware_noise_suppression */);
+    EXPECT_TRUE(result.HasValue());
+    EXPECT_EQ(result.device_id(), kUnusedDeviceID);
+    EXPECT_FALSE(
+        result.audio_processing_properties().enable_sw_echo_cancellation);
+  }
+
+  {
+    constraint_factory_.Reset();
+    constraint_factory_.basic().echo_cancellation.SetExact(true);
+    auto result = SelectSettingsAudioCapture(
+        capabilities, constraint_factory_.CreateWebMediaConstraints(),
+        false /* should_disable_hardware_noise_suppression */);
+    EXPECT_TRUE(result.HasValue());
+    EXPECT_EQ(result.device_id(), processed_source->device().id);
+    EXPECT_TRUE(
+        result.audio_processing_properties().enable_sw_echo_cancellation);
+  }
+}
+
 INSTANTIATE_TEST_CASE_P(,
                         MediaStreamConstraintsUtilAudioTest,
                         testing::Values("",

content/renderer/media/user_media_processor.cc

@@ -372,8 +372,8 @@ void UserMediaProcessor::SetupAudioInput() {
       current_request_info_->web_request().AudioConstraints(), &audio_controls);
   if (IsDeviceSource(audio_controls.stream_source)) {
     GetMediaDevicesDispatcher()->GetAudioInputCapabilities(base::BindOnce(
-        &UserMediaProcessor::SelectAudioSettings, weak_factory_.GetWeakPtr(),
-        current_request_info_->web_request()));
+        &UserMediaProcessor::SelectAudioDeviceSettings,
+        weak_factory_.GetWeakPtr(), current_request_info_->web_request()));
   } else {
     if (!IsValidAudioContentSource(audio_controls.stream_source)) {
       blink::WebString failed_constraint_name =
@@ -386,14 +386,41 @@ void UserMediaProcessor::SetupAudioInput() {
       return;
     }
     SelectAudioSettings(current_request_info_->web_request(),
-                        AudioDeviceCaptureCapabilities());
+                        {AudioDeviceCaptureCapability()});
   }
 }
 
-void UserMediaProcessor::SelectAudioSettings(
+void UserMediaProcessor::SelectAudioDeviceSettings(
     const blink::WebUserMediaRequest& web_request,
     std::vector<blink::mojom::AudioInputDeviceCapabilitiesPtr>
         audio_input_capabilities) {
+  AudioDeviceCaptureCapabilities capabilities;
+  for (const auto& device : audio_input_capabilities) {
+    MediaStreamAudioSource* audio_source = nullptr;
+    auto it =
+        std::find_if(local_sources_.begin(), local_sources_.end(),
+                     [&device](const blink::WebMediaStreamSource& web_source) {
+                       DCHECK(!web_source.IsNull());
+                       return web_source.Id().Utf8() == device->device_id;
+                     });
+    if (it != local_sources_.end()) {
+      MediaStreamSource* const source =
+          static_cast<MediaStreamSource*>(it->GetExtraData());
+      if (source->device().type == MEDIA_DEVICE_AUDIO_CAPTURE)
+        audio_source = static_cast<MediaStreamAudioSource*>(source);
+    }
+    if (audio_source)
+      capabilities.emplace_back(audio_source);
+    else
+      capabilities.emplace_back(device->device_id, device->parameters);
+  }
+
+  SelectAudioSettings(web_request, capabilities);
+}
+
+void UserMediaProcessor::SelectAudioSettings(
+    const blink::WebUserMediaRequest& web_request,
+    const std::vector<AudioDeviceCaptureCapability>& capabilities) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
   // The frame might reload or |web_request| might be cancelled while
   // capabilities are queried. Do nothing if a different request is being
@@ -403,7 +430,7 @@ void UserMediaProcessor::SelectAudioSettings(
 
   DCHECK(current_request_info_->stream_controls()->audio.requested);
   auto settings = SelectSettingsAudioCapture(
-      std::move(audio_input_capabilities), web_request.AudioConstraints(),
+      capabilities, web_request.AudioConstraints(),
       web_request.ShouldDisableHardwareNoiseSuppression());
   if (!settings.HasValue()) {
     blink::WebString failed_constraint_name =

content/renderer/media/user_media_processor.h

@@ -32,6 +32,7 @@ class WebString;
 namespace content {
 
 class AudioCaptureSettings;
+class AudioDeviceCaptureCapability;
 class MediaStreamAudioSource;
 class MediaStreamDeviceObserver;
 class MediaStreamVideoSource;
@@ -239,10 +240,13 @@ class CONTENT_EXPORT UserMediaProcessor
   GetMediaDevicesDispatcher();
 
   void SetupAudioInput();
-  void SelectAudioSettings(
+  void SelectAudioDeviceSettings(
       const blink::WebUserMediaRequest& web_request,
       std::vector<blink::mojom::AudioInputDeviceCapabilitiesPtr>
           audio_input_capabilities);
+  void SelectAudioSettings(
+      const blink::WebUserMediaRequest& web_request,
+      const std::vector<AudioDeviceCaptureCapability>& capabilities);
 
   void SetupVideoInput();
   void SelectVideoDeviceSettings(

content/test/data/media/getusermedia.html

@@ -829,6 +829,34 @@
       reportTestSuccess();
     }).catch(failTest);
   }
+
+  function getUserMediaEchoCancellationOnAndOff() {
+    var stream;
+    navigator.mediaDevices.getUserMedia({
+      audio: {
+        deviceId: {exact: "default"},
+        echoCancellation: {exact: true}
+      }
+    }).then(s => {
+      stream = s;
+      assertTrue(stream.getAudioTracks()[0].getSettings().echoCancellation);
+      // Chromium's current implementation does not support tracks from the
+      // same source to have different echo cancellation settings. This
+      // getUserMedia() call is expected to fail.
+      navigator.mediaDevices.getUserMedia({
+        audio: {
+          deviceId: {exact: "default"},
+          echoCancellation: {exact: false}
+        }
+      }).then(
+          s => failTest("Unexpected accepted promise from getUserMedia()"),
+          e => {
+            assertEquals(e.name, "OverconstrainedError");
+            stream.getAudioTracks()[0].stop();
+            reportTestSuccess();
+          });
+    }).catch(failTest);
+  }
   </script>
 </head>
 <body>