[getUserMedia] Simplify constraints processing on applyConstraints.

When applyConstraints is called on a track, the capabilities are limited
to what the settings already set on the track. This also means that the
constraints algorithm needs to guarantee that the same
processing-properties type is selected.

So far, we have done so by allowing a single processing-properties group to
be initialized when the capabilities are set. This approach is, however,
incompatible with applyConstraints reconfiguration.

This CL guarantees the same requirements, but simplifying the logic and
always initializing all the processing-based groups for processing. It
also adds the possibility to inspect the processing type selected by
returning the extra field when the select-settings algorithm is run.

BUG=796964

Change-Id: Ib34f7d33eb3aa8e1460aebcda20941df0503fa62
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1847275
Commit-Queue: Armando Miraglia <armax@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#707385}

third_party/blink/public/web/modules/mediastream/media_stream_constraints_util.h

@@ -190,6 +190,7 @@ class BLINK_MODULES_EXPORT AudioCaptureSettings {
       const base::Optional<int>& requested_buffer_size,
       bool disable_local_echo,
       bool enable_automatic_output_device_selection,
+      ProcessingType processing_type,
       const AudioProcessingProperties& audio_processing_properties);
   AudioCaptureSettings(const AudioCaptureSettings& other);
   AudioCaptureSettings& operator=(const AudioCaptureSettings& other);
@@ -216,6 +217,10 @@ class BLINK_MODULES_EXPORT AudioCaptureSettings {
     DCHECK(HasValue());
     return render_to_associated_sink_;
   }
+  ProcessingType processing_type() const {
+    DCHECK(HasValue());
+    return processing_type_;
+  }
   AudioProcessingProperties audio_processing_properties() const {
     DCHECK(HasValue());
     return audio_processing_properties_;
@@ -227,6 +232,7 @@ class BLINK_MODULES_EXPORT AudioCaptureSettings {
   base::Optional<int> requested_buffer_size_;
   bool disable_local_echo_;
   bool render_to_associated_sink_;
+  ProcessingType processing_type_;
   AudioProcessingProperties audio_processing_properties_;
 };
 

third_party/blink/renderer/modules/mediastream/media_stream_constraints_util.cc

@@ -154,12 +154,14 @@ AudioCaptureSettings::AudioCaptureSettings(
     const base::Optional<int>& requested_buffer_size,
     bool disable_local_echo,
     bool enable_automatic_output_device_selection,
+    ProcessingType processing_type,
     const AudioProcessingProperties& audio_processing_properties)
     : failed_constraint_name_(nullptr),
       device_id_(std::move(device_id)),
       requested_buffer_size_(requested_buffer_size),
       disable_local_echo_(disable_local_echo),
       render_to_associated_sink_(enable_automatic_output_device_selection),
+      processing_type_(processing_type),
       audio_processing_properties_(audio_processing_properties) {}
 
 AudioCaptureSettings::AudioCaptureSettings(const AudioCaptureSettings& other) =

third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio.cc

@@ -795,6 +795,8 @@ class ProcessingBasedContainer {
            sample_rate_container_.IsEmpty() || latency_container_.IsEmpty();
   }
 
+  ProcessingType processing_type() const { return processing_type_; }
+
  private:
   enum BooleanContainerId {
     kGoogAudioMirroring,
@@ -996,22 +998,14 @@ class DeviceContainer {
     // Three variations of the processing-based container. Each variant is
     // associated to a different type of audio processing configuration, namely
     // unprocessed, processed by WebRTC, or processed by other means.
-    if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone ||
-        source_info.type() == SourceType::kUnprocessed) {
     processing_based_containers_.push_back(
         ProcessingBasedContainer::CreateUnprocessedContainer(
             source_info, is_device_capture, device_parameters_,
             is_reconfiguration_allowed));
-    }
-    if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone ||
-        source_info.type() == SourceType::kNoApmProcessed) {
     processing_based_containers_.push_back(
         ProcessingBasedContainer::CreateNoApmProcessedContainer(
             source_info, is_device_capture, device_parameters_,
             is_reconfiguration_allowed));
-    }
-    if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone ||
-        source_info.type() == SourceType::kApmProcessed) {
     if (media::IsWebRtcApmInAudioServiceEnabled()) {
       processing_based_containers_.push_back(
           ProcessingBasedContainer::CreateRemoteApmProcessedContainer(
@@ -1023,14 +1017,8 @@ class DeviceContainer {
               source_info, is_device_capture, device_parameters_,
               is_reconfiguration_allowed));
     }
-    }
 
-#if DCHECK_IS_ON()
-    if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone)
     DCHECK_EQ(processing_based_containers_.size(), 3u);
-    else
-      DCHECK_EQ(processing_based_containers_.size(), 1u);
-#endif
 
     if (source_info.type() == SourceType::kNone)
       return;
@@ -1164,7 +1152,8 @@ class DeviceContainer {
     return std::make_tuple(
         score, AudioCaptureSettings(
                    device_id, best_requested_buffer_size, disable_local_echo,
-                   render_to_associated_sink, best_properties));
+                   render_to_associated_sink, best_container->processing_type(),
+                   best_properties));
   }
 
   // The DeviceContainer is considered empty if at least one of the

third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio_test.cc

@@ -33,6 +33,7 @@ namespace blink {
 using blink::AudioCaptureSettings;
 using blink::AudioProcessingProperties;
 using EchoCancellationType = AudioProcessingProperties::EchoCancellationType;
+using ProcessingType = AudioCaptureSettings::ProcessingType;
 
 namespace {
 
@@ -60,6 +61,15 @@ using AudioSettingsBoolMembers =
 using AudioPropertiesBoolMembers =
     std::vector<bool AudioProcessingProperties::*>;
 
+const AudioPropertiesBoolMembers kAudioProcessingProperties = {
+    &AudioProcessingProperties::goog_audio_mirroring,
+    &AudioProcessingProperties::goog_auto_gain_control,
+    &AudioProcessingProperties::goog_experimental_echo_cancellation,
+    &AudioProcessingProperties::goog_noise_suppression,
+    &AudioProcessingProperties::goog_experimental_noise_suppression,
+    &AudioProcessingProperties::goog_highpass_filter,
+    &AudioProcessingProperties::goog_experimental_auto_gain_control};
+
 template <typename T>
 static bool Contains(const std::vector<T>& vector, T value) {
   return base::Contains(vector, value);
@@ -303,6 +313,31 @@ class MediaStreamConstraintsUtilAudioTestBase {
     }
   }
 
+  void CheckProcessingType(const AudioCaptureSettings& result) {
+    ProcessingType expected_type = ProcessingType::kUnprocessed;
+    const auto& properties = result.audio_processing_properties();
+    bool properties_value = false;
+    // Skip audio mirroring and start directly from auto gain control.
+    for (size_t i = 1; i < kAudioProcessingProperties.size(); ++i)
+      properties_value |= properties.*kAudioProcessingProperties[i];
+
+    // If goog_audio_mirroring is true but all the other properties are false,
+    // we should be expecting kProcessed, however if any of the properties was
+    // true, we should expected kApmProcessed.
+    if (properties.goog_audio_mirroring && !properties_value)
+      expected_type = ProcessingType::kNoApmProcessed;
+    else if (properties_value)
+      expected_type = ProcessingType::kApmProcessed;
+
+    // Finally, if the chosen echo cancellation type is either AEC3 or AEC2, the
+    // only possible processing type to expect is kWebRtcProcessed.
+    if (properties.echo_cancellation_type ==
+        EchoCancellationType::kEchoCancellationAec3) {
+      expected_type = ProcessingType::kApmProcessed;
+    }
+    EXPECT_EQ(result.processing_type(), expected_type);
+  }
+
   void CheckDevice(const AudioDeviceCaptureCapability& expected_device,
                    const AudioCaptureSettings& result) {
     EXPECT_EQ(expected_device.DeviceID().Utf8(), result.device_id());
@@ -319,6 +354,7 @@ class MediaStreamConstraintsUtilAudioTestBase {
       const AudioSettingsBoolMembers& exclude_main_settings,
       const AudioPropertiesBoolMembers& exclude_audio_properties,
       const AudioCaptureSettings& result) {
+    CheckProcessingType(result);
     CheckBoolDefaults(exclude_main_settings, exclude_audio_properties, result);
     CheckEchoCancellationTypeDefault(result);
     CheckDeviceDefaults(result);
@@ -576,15 +612,6 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, SingleBoolConstraint) {
     }
   }
 
-  const AudioPropertiesBoolMembers kAudioProcessingProperties = {
-      &AudioProcessingProperties::goog_audio_mirroring,
-      &AudioProcessingProperties::goog_auto_gain_control,
-      &AudioProcessingProperties::goog_experimental_echo_cancellation,
-      &AudioProcessingProperties::goog_noise_suppression,
-      &AudioProcessingProperties::goog_experimental_noise_suppression,
-      &AudioProcessingProperties::goog_highpass_filter,
-      &AudioProcessingProperties::goog_experimental_auto_gain_control};
-
   const std::vector<
       blink::BooleanConstraint blink::WebMediaTrackConstraintSet::*>
       kAudioProcessingConstraints = {
@@ -1161,6 +1188,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, IdealArbitraryDeviceID) {
     CheckDeviceDefaults(result);
   else
     EXPECT_EQ(kArbitraryDeviceID, result.device_id());
+  CheckProcessingType(result);
   CheckBoolDefaults(AudioSettingsBoolMembers(), AudioPropertiesBoolMembers(),
                     result);
   CheckEchoCancellationTypeDefault(result);
@@ -1173,6 +1201,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, ExactValidDeviceID) {
     auto result = SelectSettings();
     EXPECT_TRUE(result.HasValue());
     CheckDevice(device, result);
+    CheckProcessingType(result);
     CheckBoolDefaults(AudioSettingsBoolMembers(), AudioPropertiesBoolMembers(),
                       result);
     EchoCancellationType expected_echo_cancellation_type =
@@ -1198,6 +1227,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, ExactGroupID) {
     auto result = SelectSettings();
     EXPECT_TRUE(result.HasValue());
     CheckDevice(device, result);
+    CheckProcessingType(result);
     CheckBoolDefaults(AudioSettingsBoolMembers(), AudioPropertiesBoolMembers(),
                       result);
     EchoCancellationType expected_echo_cancellation_type =
@@ -1266,6 +1296,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, EchoCancellationWithWebRtc) {
         EXPECT_EQ(GetMediaStreamSource() != blink::kMediaStreamSourceDesktop,
                   result.disable_local_echo());
         EXPECT_FALSE(result.render_to_associated_sink());
+        CheckProcessingType(result);
         if (IsDeviceCapture()) {
           CheckDevice(*default_device_, result);
         } else {
@@ -1324,6 +1355,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, EchoCancellationWithSystem) {
         EXPECT_EQ(GetMediaStreamSource() != blink::kMediaStreamSourceDesktop,
                   result.disable_local_echo());
         EXPECT_FALSE(result.render_to_associated_sink());
+        CheckProcessingType(result);
         CheckDevice(*system_echo_canceller_device_, result);
       }
     }
@@ -1358,6 +1390,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, GoogEchoCancellationWithWebRtc) {
                   : EchoCancellationType::kEchoCancellationDisabled;
         EXPECT_EQ(expected_echo_cancellation_type,
                   properties.echo_cancellation_type);
+        CheckProcessingType(result);
         CheckBoolDefaults(AudioSettingsBoolMembers(),
                           AudioPropertiesBoolMembers(), result);
         if (IsDeviceCapture()) {
@@ -1405,6 +1438,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, GoogEchoCancellationWithSystem) {
                   : EchoCancellationType::kEchoCancellationDisabled;
         EXPECT_EQ(expected_echo_cancellation_type,
                   properties.echo_cancellation_type);
+        CheckProcessingType(result);
         CheckBoolDefaults(AudioSettingsBoolMembers(),
                           AudioPropertiesBoolMembers(), result);
         CheckDevice(*system_echo_canceller_device_, result);
@@ -1430,14 +1464,6 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, ContradictoryEchoCancellation) {
 // default value set by the echoCancellation constraint.
 TEST_P(MediaStreamConstraintsUtilAudioTest,
        EchoCancellationAndSingleBoolConstraint) {
-  const AudioPropertiesBoolMembers kAudioProcessingProperties = {
-      &AudioProcessingProperties::goog_audio_mirroring,
-      &AudioProcessingProperties::goog_auto_gain_control,
-      &AudioProcessingProperties::goog_experimental_echo_cancellation,
-      &AudioProcessingProperties::goog_noise_suppression,
-      &AudioProcessingProperties::goog_experimental_noise_suppression,
-      &AudioProcessingProperties::goog_highpass_filter,
-      &AudioProcessingProperties::goog_experimental_auto_gain_control};
 
   const std::vector<
       blink::BooleanConstraint blink::WebMediaTrackConstraintSet::*>
@@ -1473,6 +1499,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest,
          set_function)(true);
         auto result = SelectSettings();
         EXPECT_TRUE(result.HasValue());
+        CheckProcessingType(result);
         EXPECT_EQ(EchoCancellationType::kEchoCancellationDisabled,
                   result.audio_processing_properties().echo_cancellation_type);
         EXPECT_TRUE(result.audio_processing_properties().*
@@ -1531,6 +1558,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, AdvancedCompatibleConstraints) {
   auto result = SelectSettings();
   EXPECT_TRUE(result.HasValue());
   CheckDeviceDefaults(result);
+  CheckProcessingType(result);
   CheckBoolDefaults({&AudioCaptureSettings::render_to_associated_sink},
                     {&AudioProcessingProperties::goog_audio_mirroring}, result);
   CheckEchoCancellationTypeDefault(result);
@@ -1549,6 +1577,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest,
   constraint_factory_.AddAdvanced().goog_audio_mirroring.SetExact(true);
   auto result = SelectSettings();
   EXPECT_TRUE(result.HasValue());
+  CheckProcessingType(result);
   CheckDeviceDefaults(result);
   CheckBoolDefaults({},
                     {&AudioProcessingProperties::goog_audio_mirroring,
@@ -1566,6 +1595,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, AdvancedConflictingLastConstraint) {
   constraint_factory_.AddAdvanced().goog_audio_mirroring.SetExact(true);
   auto result = SelectSettings();
   EXPECT_TRUE(result.HasValue());
+  CheckProcessingType(result);
   CheckDeviceDefaults(result);
   CheckBoolDefaults({},
                     {&AudioProcessingProperties::goog_audio_mirroring,
@@ -1695,15 +1725,6 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, SourceWithAudioProcessing) {
             &blink::WebMediaTrackConstraintSet::
                 goog_experimental_auto_gain_control,
         };
-    const AudioPropertiesBoolMembers kAudioProcessingProperties = {
-        &AudioProcessingProperties::goog_audio_mirroring,
-        &AudioProcessingProperties::goog_auto_gain_control,
-        &AudioProcessingProperties::goog_experimental_echo_cancellation,
-        &AudioProcessingProperties::goog_noise_suppression,
-        &AudioProcessingProperties::goog_experimental_noise_suppression,
-        &AudioProcessingProperties::goog_highpass_filter,
-        &AudioProcessingProperties::goog_experimental_auto_gain_control};
-
     ASSERT_EQ(kAudioProcessingConstraints.size(),
               kAudioProcessingProperties.size());