Skip to content

GitLab

T
tangled
Commit e19ba596 authored by Armando Miraglia's avatar Armando Miraglia Committed by Commit Bot
Browse files
Options

[getUserMedia] Simplify constraints processing on applyConstraints. 

When applyConstraints is called on a track, the capabilities are limited
to what the settings already set on the track. This also means that the
constraints algorithm needs to guarantee that the same
processing-properties type is selected.

So far, we have done so by allowing a single processing-properties group to
be initialized when the capabilities are set. This approach is, however,
incompatible with applyConstraints reconfiguration.

This CL guarantees the same requirements, but simplifying the logic and
always initializing all the processing-based groups for processing. It
also adds the possibility to inspect the processing type selected by
returning the extra field when the select-settings algorithm is run.

BUG=796964

Change-Id: Ib34f7d33eb3aa8e1460aebcda20941df0503fa62
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1847275
Commit-Queue: Armando Miraglia <armax@chromium.org>
Reviewed-by: default avatarGuido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#707385}
parent 97dbb8fa
Hide whitespace changes
Inline Side-by-side
Showing with 72 additions and 54 deletions
third_party/blink/public/web/modules/mediastream/media_stream_constraints_util.h
View file @ e19ba596
......@@ -190,6 +190,7 @@ class BLINK_MODULES_EXPORT AudioCaptureSettings {
const base::Optional<int>& requested_buffer_size,
bool disable_local_echo,
bool enable_automatic_output_device_selection,
ProcessingType processing_type,
const AudioProcessingProperties& audio_processing_properties);
AudioCaptureSettings(const AudioCaptureSettings& other);
AudioCaptureSettings& operator=(const AudioCaptureSettings& other);
......@@ -216,6 +217,10 @@ class BLINK_MODULES_EXPORT AudioCaptureSettings {
DCHECK(HasValue());
return render_to_associated_sink_;
}
ProcessingType processing_type() const {
DCHECK(HasValue());
return processing_type_;
}
AudioProcessingProperties audio_processing_properties() const {
DCHECK(HasValue());
return audio_processing_properties_;
......@@ -227,6 +232,7 @@ class BLINK_MODULES_EXPORT AudioCaptureSettings {
base::Optional<int> requested_buffer_size_;
bool disable_local_echo_;
bool render_to_associated_sink_;
ProcessingType processing_type_;
AudioProcessingProperties audio_processing_properties_;
};
......
third_party/blink/renderer/modules/mediastream/media_stream_constraints_util.cc
View file @ e19ba596
......@@ -154,12 +154,14 @@ AudioCaptureSettings::AudioCaptureSettings(
const base::Optional<int>& requested_buffer_size,
bool disable_local_echo,
bool enable_automatic_output_device_selection,
ProcessingType processing_type,
const AudioProcessingProperties& audio_processing_properties)
: failed_constraint_name_(nullptr),
device_id_(std::move(device_id)),
requested_buffer_size_(requested_buffer_size),
disable_local_echo_(disable_local_echo),
render_to_associated_sink_(enable_automatic_output_device_selection),
processing_type_(processing_type),
audio_processing_properties_(audio_processing_properties) {}
AudioCaptureSettings::AudioCaptureSettings(const AudioCaptureSettings& other) =
......
third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio.cc
View file @ e19ba596
......@@ -795,6 +795,8 @@ class ProcessingBasedContainer {
sample_rate_container_.IsEmpty() || latency_container_.IsEmpty();
}
ProcessingType processing_type() const { return processing_type_; }
private:
enum BooleanContainerId {
kGoogAudioMirroring,
......@@ -996,41 +998,27 @@ class DeviceContainer {
// Three variations of the processing-based container. Each variant is
// associated to a different type of audio processing configuration, namely
// unprocessed, processed by WebRTC, or processed by other means.
if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone ||
source_info.type() == SourceType::kUnprocessed) {
processing_based_containers_.push_back(
ProcessingBasedContainer::CreateUnprocessedContainer(
source_info, is_device_capture, device_parameters_,
is_reconfiguration_allowed));
processing_based_containers_.push_back(
ProcessingBasedContainer::CreateNoApmProcessedContainer(
source_info, is_device_capture, device_parameters_,
is_reconfiguration_allowed));
if (media::IsWebRtcApmInAudioServiceEnabled()) {
processing_based_containers_.push_back(
ProcessingBasedContainer::CreateUnprocessedContainer(
ProcessingBasedContainer::CreateRemoteApmProcessedContainer(
source_info, is_device_capture, device_parameters_,
is_reconfiguration_allowed));
}
if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone ||
source_info.type() == SourceType::kNoApmProcessed) {
} else {
processing_based_containers_.push_back(
ProcessingBasedContainer::CreateNoApmProcessedContainer(
ProcessingBasedContainer::CreateApmProcessedContainer(
source_info, is_device_capture, device_parameters_,
is_reconfiguration_allowed));
}
if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone ||
source_info.type() == SourceType::kApmProcessed) {
if (media::IsWebRtcApmInAudioServiceEnabled()) {
processing_based_containers_.push_back(
ProcessingBasedContainer::CreateRemoteApmProcessedContainer(
source_info, is_device_capture, device_parameters_,
is_reconfiguration_allowed));
} else {
processing_based_containers_.push_back(
ProcessingBasedContainer::CreateApmProcessedContainer(
source_info, is_device_capture, device_parameters_,
is_reconfiguration_allowed));
}
}
#if DCHECK_IS_ON()
if (is_reconfiguration_allowed || source_info.type() == SourceType::kNone)
DCHECK_EQ(processing_based_containers_.size(), 3u);
else
DCHECK_EQ(processing_based_containers_.size(), 1u);
#endif
DCHECK_EQ(processing_based_containers_.size(), 3u);
if (source_info.type() == SourceType::kNone)
return;
......@@ -1164,7 +1152,8 @@ class DeviceContainer {
return std::make_tuple(
score, AudioCaptureSettings(
device_id, best_requested_buffer_size, disable_local_echo,
render_to_associated_sink, best_properties));
render_to_associated_sink, best_container->processing_type(),
best_properties));
}
// The DeviceContainer is considered empty if at least one of the
......
third_party/blink/renderer/modules/mediastream/media_stream_constraints_util_audio_test.cc
View file @ e19ba596
......@@ -33,6 +33,7 @@ namespace blink {
using blink::AudioCaptureSettings;
using blink::AudioProcessingProperties;
using EchoCancellationType = AudioProcessingProperties::EchoCancellationType;
using ProcessingType = AudioCaptureSettings::ProcessingType;
namespace {
......@@ -60,6 +61,15 @@ using AudioSettingsBoolMembers =
using AudioPropertiesBoolMembers =
std::vector<bool AudioProcessingProperties::*>;
const AudioPropertiesBoolMembers kAudioProcessingProperties = {
&AudioProcessingProperties::goog_audio_mirroring,
&AudioProcessingProperties::goog_auto_gain_control,
&AudioProcessingProperties::goog_experimental_echo_cancellation,
&AudioProcessingProperties::goog_noise_suppression,
&AudioProcessingProperties::goog_experimental_noise_suppression,
&AudioProcessingProperties::goog_highpass_filter,
&AudioProcessingProperties::goog_experimental_auto_gain_control};
template <typename T>
static bool Contains(const std::vector<T>& vector, T value) {
return base::Contains(vector, value);
......@@ -303,6 +313,31 @@ class MediaStreamConstraintsUtilAudioTestBase {
}
}
void CheckProcessingType(const AudioCaptureSettings& result) {
ProcessingType expected_type = ProcessingType::kUnprocessed;
const auto& properties = result.audio_processing_properties();
bool properties_value = false;
// Skip audio mirroring and start directly from auto gain control.
for (size_t i = 1; i < kAudioProcessingProperties.size(); ++i)
properties_value |= properties.*kAudioProcessingProperties[i];
// If goog_audio_mirroring is true but all the other properties are false,
// we should be expecting kProcessed, however if any of the properties was
// true, we should expected kApmProcessed.
if (properties.goog_audio_mirroring && !properties_value)
expected_type = ProcessingType::kNoApmProcessed;
else if (properties_value)
expected_type = ProcessingType::kApmProcessed;
// Finally, if the chosen echo cancellation type is either AEC3 or AEC2, the
// only possible processing type to expect is kWebRtcProcessed.
if (properties.echo_cancellation_type ==
EchoCancellationType::kEchoCancellationAec3) {
expected_type = ProcessingType::kApmProcessed;
}
EXPECT_EQ(result.processing_type(), expected_type);
}
void CheckDevice(const AudioDeviceCaptureCapability& expected_device,
const AudioCaptureSettings& result) {
EXPECT_EQ(expected_device.DeviceID().Utf8(), result.device_id());
......@@ -319,6 +354,7 @@ class MediaStreamConstraintsUtilAudioTestBase {
const AudioSettingsBoolMembers& exclude_main_settings,
const AudioPropertiesBoolMembers& exclude_audio_properties,
const AudioCaptureSettings& result) {
CheckProcessingType(result);
CheckBoolDefaults(exclude_main_settings, exclude_audio_properties, result);
CheckEchoCancellationTypeDefault(result);
CheckDeviceDefaults(result);
......@@ -576,15 +612,6 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, SingleBoolConstraint) {
}
}
const AudioPropertiesBoolMembers kAudioProcessingProperties = {
&AudioProcessingProperties::goog_audio_mirroring,
&AudioProcessingProperties::goog_auto_gain_control,
&AudioProcessingProperties::goog_experimental_echo_cancellation,
&AudioProcessingProperties::goog_noise_suppression,
&AudioProcessingProperties::goog_experimental_noise_suppression,
&AudioProcessingProperties::goog_highpass_filter,
&AudioProcessingProperties::goog_experimental_auto_gain_control};
const std::vector<
blink::BooleanConstraint blink::WebMediaTrackConstraintSet::*>
kAudioProcessingConstraints = {
......@@ -1161,6 +1188,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, IdealArbitraryDeviceID) {
CheckDeviceDefaults(result);
else
EXPECT_EQ(kArbitraryDeviceID, result.device_id());
CheckProcessingType(result);
CheckBoolDefaults(AudioSettingsBoolMembers(), AudioPropertiesBoolMembers(),
result);
CheckEchoCancellationTypeDefault(result);
......@@ -1173,6 +1201,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, ExactValidDeviceID) {
auto result = SelectSettings();
EXPECT_TRUE(result.HasValue());
CheckDevice(device, result);
CheckProcessingType(result);
CheckBoolDefaults(AudioSettingsBoolMembers(), AudioPropertiesBoolMembers(),
result);
EchoCancellationType expected_echo_cancellation_type =
......@@ -1198,6 +1227,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, ExactGroupID) {
auto result = SelectSettings();
EXPECT_TRUE(result.HasValue());
CheckDevice(device, result);
CheckProcessingType(result);
CheckBoolDefaults(AudioSettingsBoolMembers(), AudioPropertiesBoolMembers(),
result);
EchoCancellationType expected_echo_cancellation_type =
......@@ -1266,6 +1296,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, EchoCancellationWithWebRtc) {
EXPECT_EQ(GetMediaStreamSource() != blink::kMediaStreamSourceDesktop,
result.disable_local_echo());
EXPECT_FALSE(result.render_to_associated_sink());
CheckProcessingType(result);
if (IsDeviceCapture()) {
CheckDevice(*default_device_, result);
} else {
......@@ -1324,6 +1355,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, EchoCancellationWithSystem) {
EXPECT_EQ(GetMediaStreamSource() != blink::kMediaStreamSourceDesktop,
result.disable_local_echo());
EXPECT_FALSE(result.render_to_associated_sink());
CheckProcessingType(result);
CheckDevice(*system_echo_canceller_device_, result);
}
}
......@@ -1358,6 +1390,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, GoogEchoCancellationWithWebRtc) {
: EchoCancellationType::kEchoCancellationDisabled;
EXPECT_EQ(expected_echo_cancellation_type,
properties.echo_cancellation_type);
CheckProcessingType(result);
CheckBoolDefaults(AudioSettingsBoolMembers(),
AudioPropertiesBoolMembers(), result);
if (IsDeviceCapture()) {
......@@ -1405,6 +1438,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, GoogEchoCancellationWithSystem) {
: EchoCancellationType::kEchoCancellationDisabled;
EXPECT_EQ(expected_echo_cancellation_type,
properties.echo_cancellation_type);
CheckProcessingType(result);
CheckBoolDefaults(AudioSettingsBoolMembers(),
AudioPropertiesBoolMembers(), result);
CheckDevice(*system_echo_canceller_device_, result);
......@@ -1430,14 +1464,6 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, ContradictoryEchoCancellation) {
// default value set by the echoCancellation constraint.
TEST_P(MediaStreamConstraintsUtilAudioTest,
EchoCancellationAndSingleBoolConstraint) {
const AudioPropertiesBoolMembers kAudioProcessingProperties = {
&AudioProcessingProperties::goog_audio_mirroring,
&AudioProcessingProperties::goog_auto_gain_control,
&AudioProcessingProperties::goog_experimental_echo_cancellation,
&AudioProcessingProperties::goog_noise_suppression,
&AudioProcessingProperties::goog_experimental_noise_suppression,
&AudioProcessingProperties::goog_highpass_filter,
&AudioProcessingProperties::goog_experimental_auto_gain_control};
const std::vector<
blink::BooleanConstraint blink::WebMediaTrackConstraintSet::*>
......@@ -1473,6 +1499,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest,
set_function)(true);
auto result = SelectSettings();
EXPECT_TRUE(result.HasValue());
CheckProcessingType(result);
EXPECT_EQ(EchoCancellationType::kEchoCancellationDisabled,
result.audio_processing_properties().echo_cancellation_type);
EXPECT_TRUE(result.audio_processing_properties().*
......@@ -1531,6 +1558,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, AdvancedCompatibleConstraints) {
auto result = SelectSettings();
EXPECT_TRUE(result.HasValue());
CheckDeviceDefaults(result);
CheckProcessingType(result);
CheckBoolDefaults({&AudioCaptureSettings::render_to_associated_sink},
{&AudioProcessingProperties::goog_audio_mirroring}, result);
CheckEchoCancellationTypeDefault(result);
......@@ -1549,6 +1577,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest,
constraint_factory_.AddAdvanced().goog_audio_mirroring.SetExact(true);
auto result = SelectSettings();
EXPECT_TRUE(result.HasValue());
CheckProcessingType(result);
CheckDeviceDefaults(result);
CheckBoolDefaults({},
{&AudioProcessingProperties::goog_audio_mirroring,
......@@ -1566,6 +1595,7 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, AdvancedConflictingLastConstraint) {
constraint_factory_.AddAdvanced().goog_audio_mirroring.SetExact(true);
auto result = SelectSettings();
EXPECT_TRUE(result.HasValue());
CheckProcessingType(result);
CheckDeviceDefaults(result);
CheckBoolDefaults({},
{&AudioProcessingProperties::goog_audio_mirroring,
......@@ -1695,15 +1725,6 @@ TEST_P(MediaStreamConstraintsUtilAudioTest, SourceWithAudioProcessing) {
&blink::WebMediaTrackConstraintSet::
goog_experimental_auto_gain_control,
};
const AudioPropertiesBoolMembers kAudioProcessingProperties = {
&AudioProcessingProperties::goog_audio_mirroring,
&AudioProcessingProperties::goog_auto_gain_control,
&AudioProcessingProperties::goog_experimental_echo_cancellation,
&AudioProcessingProperties::goog_noise_suppression,
&AudioProcessingProperties::goog_experimental_noise_suppression,
&AudioProcessingProperties::goog_highpass_filter,
&AudioProcessingProperties::goog_experimental_auto_gain_control};
ASSERT_EQ(kAudioProcessingConstraints.size(),
kAudioProcessingProperties.size());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment