Commit 4df8e0bd authored by Ken MacKay's avatar Ken MacKay Committed by Kenneth MacKay

[Chromecast] Refactor playout channel selection

Move playout channel selection and mono mixing to StreamMixer to
simplify the logic.

Bug: internal b/118685514
Change-Id: I6c2d871d23958d615c4bf828b5b6013929e3bea0
Reviewed-on: https://chromium-review.googlesource.com/c/1347207Reviewed-by: default avatarLuke Halliwell <halliwell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#610991}
parent 226888c4
......@@ -19,15 +19,11 @@ namespace chromecast {
namespace media {
FilterGroup::FilterGroup(int num_channels,
GroupType type,
const std::string& name,
std::unique_ptr<PostProcessingPipeline> pipeline,
const base::flat_set<std::string>& device_ids,
const std::vector<FilterGroup*>& mixed_inputs)
: num_channels_(num_channels),
type_(type),
mix_to_mono_(false),
playout_channel_(kChannelAll),
name_(name),
device_ids_(device_ids),
mixed_inputs_(mixed_inputs),
......@@ -145,19 +141,6 @@ float FilterGroup::MixAndFilter(
}
}
// Copy the active channel to all channels. Only used in the "linearize"
// instance.
if (playout_channel_ != kChannelAll && type_ == GroupType::kLinearize) {
DCHECK_GE(playout_channel_, 0);
DCHECK_LT(playout_channel_, num_channels_);
for (int frame = 0; frame < num_frames; ++frame) {
float s = interleaved_.get()[frame * num_channels_ + playout_channel_];
for (int c = 0; c < num_channels_; ++c)
interleaved_.get()[frame * num_channels_ + c] = s;
}
}
// Allow paused streams to "ring out" at the last valid volume.
// If the stream volume is actually 0, this doesn't matter, since the
// data is 0's anyway.
......@@ -174,18 +157,6 @@ float FilterGroup::MixAndFilter(
delay_frames_ = post_processing_pipeline_->ProcessFrames(
interleaved_.get(), num_frames, last_volume_, is_silence);
// Mono mixing if needed. Only used in the "Mix" instance.
if (mix_to_mono_ && type_ == GroupType::kFinalMix) {
for (int frame = 0; frame < num_frames; ++frame) {
float sum = 0;
for (int c = 0; c < num_channels_; ++c)
sum += interleaved_.get()[frame * num_channels_ + c];
for (int c = 0; c < num_channels_; ++c)
interleaved_.get()[frame * num_channels_ + c] = sum / num_channels_;
}
}
return last_volume_;
}
......@@ -240,18 +211,13 @@ void FilterGroup::SetPostProcessorConfig(const std::string& name,
post_processing_pipeline_->SetPostProcessorConfig(name, config);
}
void FilterGroup::SetMixToMono(bool mix_to_mono) {
mix_to_mono_ = (num_channels_ != 1 && mix_to_mono);
}
void FilterGroup::UpdatePlayoutChannel(int playout_channel) {
if (playout_channel >= num_channels_) {
LOG(ERROR) << "only " << num_channels_ << " present, wanted channel #"
<< playout_channel;
return;
}
playout_channel_ = playout_channel;
post_processing_pipeline_->UpdatePlayoutChannel(playout_channel_);
post_processing_pipeline_->UpdatePlayoutChannel(playout_channel);
}
} // namespace media
......
......@@ -36,12 +36,8 @@ class PostProcessingPipeline;
// MixAndFilter() is called (they must be added each time data is queried).
class FilterGroup {
public:
enum class GroupType { kStream, kFinalMix, kLinearize };
// |num_channels| indicates number of input audio channels.
// |type| indicates where in the pipeline this FilterGroup sits.
// some features are specific to certain locations:
// - mono mixer takes place at the end of kFinalMix.
// - channel selection occurs before post-processing in kLinearize.
// |name| is used for debug printing
// |pipeline| - processing pipeline.
// |device_ids| is a set of strings that is used as a filter to determine
......@@ -53,7 +49,6 @@ class FilterGroup {
// but there is no technical limitation preventing mixing input classes.
FilterGroup(int num_channels,
GroupType type,
const std::string& name,
std::unique_ptr<PostProcessingPipeline> pipeline,
const base::flat_set<std::string>& device_ids,
......@@ -107,10 +102,7 @@ class FilterGroup {
void SetPostProcessorConfig(const std::string& name,
const std::string& config);
// Toggles the mono mixer.
void SetMixToMono(bool mix_to_mono);
// Sets the active channel.
// Sets the active channel for post processors.
void UpdatePlayoutChannel(int playout_channel);
// Get content type
......@@ -123,9 +115,6 @@ class FilterGroup {
void AddTempBuffer(int num_channels, int num_frames);
const int num_channels_;
const GroupType type_;
bool mix_to_mono_;
int playout_channel_;
const std::string name_;
const base::flat_set<std::string> device_ids_;
std::vector<FilterGroup*> mixed_inputs_;
......
......@@ -164,17 +164,14 @@ class FilterGroupTest : public testing::Test {
~FilterGroupTest() override {}
void MakeFilterGroup(
FilterGroup::GroupType type,
bool mix_to_mono,
std::unique_ptr<MockPostProcessingPipeline> post_processor) {
post_processor_ = post_processor.get();
EXPECT_CALL(*post_processor_, SetContentType(kDefaultContentType));
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(kDefaultPlayoutChannel));
filter_group_ = std::make_unique<FilterGroup>(
kNumInputChannels, type, "test_filter", std::move(post_processor),
kNumInputChannels, "test_filter", std::move(post_processor),
base::flat_set<std::string>() /* device_ids */,
std::vector<FilterGroup*>());
filter_group_->SetMixToMono(mix_to_mono);
filter_group_->Initialize(kInputSampleRate);
filter_group_->AddInput(&input_);
filter_group_->UpdatePlayoutChannel(kChannelAll);
......@@ -210,120 +207,26 @@ class FilterGroupTest : public testing::Test {
};
TEST_F(FilterGroupTest, Passthrough) {
MakeFilterGroup(FilterGroup::GroupType::kFinalMix, false /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
MakeFilterGroup(std::make_unique<NiceMock<MockPostProcessingPipeline>>());
EXPECT_CALL(*post_processor_, ProcessFrames(_, kInputFrames, _, false));
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
}
TEST_F(FilterGroupTest, StreamGroupsDoNotMonoMix) {
MakeFilterGroup(FilterGroup::GroupType::kStream, true /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
EXPECT_CALL(*post_processor_, ProcessFrames(_, kInputFrames, _, false));
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
}
TEST_F(FilterGroupTest, LinearizeGroupsDoNotMonoMix) {
MakeFilterGroup(FilterGroup::GroupType::kLinearize, true /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
EXPECT_CALL(*post_processor_, ProcessFrames(_, kInputFrames, _, false));
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
}
TEST_F(FilterGroupTest, MonoMixer) {
MakeFilterGroup(FilterGroup::GroupType::kFinalMix, true /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
// Verify if the fiter group output matches the source after down mixing.
float* interleaved_data = filter_group_->GetOutputBuffer();
for (int i = 0; i < kInputFrames; ++i) {
ASSERT_EQ((LeftInput(i) + RightInput(i)) / 2, interleaved_data[i * 2]);
ASSERT_EQ(interleaved_data[i * 2], interleaved_data[i * 2 + 1]);
}
}
TEST_F(FilterGroupTest, MonoMixesAfterPostProcessors) {
MakeFilterGroup(FilterGroup::GroupType::kFinalMix, true /* mix to mono */,
std::make_unique<NiceMock<InvertChannelPostProcessor>>(
kNumInputChannels, 0));
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
// Verify both output channels = (-1 * ch0 + ch1) / 2 after mixing.
// If order of mixing, filtering is incorrect, the channels won't match.
float* interleaved_data = filter_group_->GetOutputBuffer();
for (int i = 0; i < kInputFrames; ++i) {
ASSERT_EQ((-LeftInput(i) + RightInput(i)) / 2, interleaved_data[i * 2]);
ASSERT_EQ(interleaved_data[i * 2], interleaved_data[i * 2 + 1]);
}
}
TEST_F(FilterGroupTest, StreamGroupDoesNotSelectChannels) {
MakeFilterGroup(FilterGroup::GroupType::kStream, false /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(0));
filter_group_->UpdatePlayoutChannel(0);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
source_.SetData(GetTestData());
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(1));
filter_group_->UpdatePlayoutChannel(1);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
}
TEST_F(FilterGroupTest, MixGroupDoesNotSelectChannels) {
MakeFilterGroup(FilterGroup::GroupType::kFinalMix, false /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(0));
filter_group_->UpdatePlayoutChannel(0);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
source_.SetData(GetTestData());
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(1));
filter_group_->UpdatePlayoutChannel(1);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
AssertPassthrough();
}
TEST_F(FilterGroupTest, SelectsOutputChannel) {
MakeFilterGroup(FilterGroup::GroupType::kLinearize, false /* mix to mono */,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
MakeFilterGroup(std::make_unique<NiceMock<MockPostProcessingPipeline>>());
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(0));
filter_group_->UpdatePlayoutChannel(0);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
float* interleaved_data = filter_group_->GetOutputBuffer();
for (int f = 0; f < kInputFrames; ++f) {
for (int ch = 0; ch < kNumInputChannels; ++ch) {
// Both output channels should be equal to left channel.
ASSERT_EQ(interleaved_data[f * kNumInputChannels + ch], LeftInput(f));
}
}
testing::Mock::VerifyAndClearExpectations(post_processor_);
source_.SetData(GetTestData());
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(1));
filter_group_->UpdatePlayoutChannel(1);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
for (int f = 0; f < kInputFrames; ++f) {
for (int ch = 0; ch < kNumInputChannels; ++ch) {
// Both output channels should be equal to right channel.
ASSERT_EQ(interleaved_data[f * kNumInputChannels + ch], RightInput(f));
}
}
testing::Mock::VerifyAndClearExpectations(post_processor_);
source_.SetData(GetTestData());
......@@ -331,35 +234,10 @@ TEST_F(FilterGroupTest, SelectsOutputChannel) {
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(-1));
filter_group_->UpdatePlayoutChannel(-1);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
for (int f = 0; f < kInputFrames; ++f) {
for (int ch = 0; ch < kNumInputChannels; ++ch) {
// Back to normal (passthrough).
ASSERT_EQ(interleaved_data[f * kNumInputChannels + ch], Input(ch, f));
}
}
}
TEST_F(FilterGroupTest, SelectsOutputChannelBeforePostProcessors) {
MakeFilterGroup(FilterGroup::GroupType::kLinearize, false /* mix to mono */,
std::make_unique<NiceMock<InvertChannelPostProcessor>>(
kNumInputChannels, 0));
EXPECT_CALL(*post_processor_, UpdatePlayoutChannel(0));
filter_group_->UpdatePlayoutChannel(0);
filter_group_->MixAndFilter(kInputFrames, RenderingDelay());
float* interleaved_data = filter_group_->GetOutputBuffer();
for (int f = 0; f < kInputFrames; ++f) {
// channel 0 out = channel 0 in * -1
// channel 1 out = channel 0 in
// (If order is wrong, both channels will be channel_0_in * -1).
ASSERT_EQ(interleaved_data[f * kNumInputChannels], LeftInput(f) * -1);
ASSERT_EQ(interleaved_data[f * kNumInputChannels + 1], LeftInput(f));
}
}
TEST_F(FilterGroupTest, ChecksContentType) {
MakeFilterGroup(FilterGroup::GroupType::kStream, false,
std::make_unique<NiceMock<MockPostProcessingPipeline>>());
MakeFilterGroup(std::make_unique<NiceMock<MockPostProcessingPipeline>>());
NiceMock<MockMixerSource> tts_source(kInputSampleRate);
tts_source.set_content_type(AudioContentType::kCommunication);
......@@ -404,8 +282,7 @@ TEST_F(FilterGroupTest, ChecksContentType) {
TEST_F(FilterGroupTest, ReportsOutputChannels) {
const int num_output_channels = 4;
MakeFilterGroup(FilterGroup::GroupType::kStream, false,
std::make_unique<NiceMock<MockPostProcessingPipeline>>(
MakeFilterGroup(std::make_unique<NiceMock<MockPostProcessingPipeline>>(
num_output_channels));
EXPECT_EQ(num_output_channels, filter_group_->GetOutputChannelCount());
......
......@@ -30,7 +30,6 @@ bool IsOutputDeviceId(const std::string& device) {
}
std::unique_ptr<FilterGroup> CreateFilterGroup(
FilterGroup::GroupType type,
int input_channels,
const std::string& name,
const base::ListValue* filter_list,
......@@ -40,9 +39,8 @@ std::unique_ptr<FilterGroup> CreateFilterGroup(
DCHECK(ppp_factory);
auto pipeline =
ppp_factory->CreatePipeline(name, filter_list, input_channels);
return std::make_unique<FilterGroup>(input_channels, type, name,
std::move(pipeline), device_ids,
mixed_inputs);
return std::make_unique<FilterGroup>(
input_channels, name, std::move(pipeline), device_ids, mixed_inputs);
}
} // namespace
......@@ -83,8 +81,8 @@ bool MixerPipeline::BuildPipeline(PostProcessingPipelineParser* config,
return false;
}
filter_groups_.push_back(CreateFilterGroup(
FilterGroup::GroupType::kStream, kNumInputChannels,
*device_ids.begin() /* name */, stream_pipeline.pipeline, device_ids,
kNumInputChannels, *device_ids.begin() /* name */,
stream_pipeline.pipeline, device_ids,
std::vector<FilterGroup*>() /* mixed_inputs */, factory));
if (device_ids.find(::media::AudioDeviceDescription::kDefaultDeviceId) !=
device_ids.end()) {
......@@ -110,8 +108,7 @@ bool MixerPipeline::BuildPipeline(PostProcessingPipelineParser* config,
}
filter_groups_.push_back(CreateFilterGroup(
FilterGroup::GroupType::kFinalMix, mix_group_input_channels, "mix",
config->GetMixPipeline(),
mix_group_input_channels, "mix", config->GetMixPipeline(),
base::flat_set<std::string>() /* device_ids */, filter_group_ptrs,
factory));
} else {
......@@ -119,8 +116,7 @@ bool MixerPipeline::BuildPipeline(PostProcessingPipelineParser* config,
std::string kDefaultDeviceId =
::media::AudioDeviceDescription::kDefaultDeviceId;
filter_groups_.push_back(CreateFilterGroup(
FilterGroup::GroupType::kFinalMix, kNumInputChannels, "mix",
config->GetMixPipeline(),
kNumInputChannels, "mix", config->GetMixPipeline(),
base::flat_set<std::string>({kDefaultDeviceId}),
std::vector<FilterGroup*>() /* mixed_inputs */, factory));
default_stream_group_ = filter_groups_.back().get();
......@@ -129,7 +125,6 @@ bool MixerPipeline::BuildPipeline(PostProcessingPipelineParser* config,
loopback_output_group_ = filter_groups_.back().get();
filter_groups_.push_back(CreateFilterGroup(
FilterGroup::GroupType::kLinearize,
loopback_output_group_->GetOutputChannelCount(), "linearize",
config->GetLinearizePipeline(),
base::flat_set<std::string>() /* device_ids */,
......@@ -206,10 +201,6 @@ void MixerPipeline::SetPostProcessorConfig(const std::string& name,
}
}
void MixerPipeline::SetMixToMono(bool mix_to_mono) {
loopback_output_group_->SetMixToMono(mix_to_mono);
}
void MixerPipeline::SetPlayoutChannel(int playout_channel) {
for (auto& filter_group : filter_groups_) {
filter_group->UpdatePlayoutChannel(playout_channel);
......
......@@ -70,9 +70,6 @@ class MixerPipeline {
// and GetOutput(), i.e. the group delay of PostProcessors in "linearize"
int64_t GetPostLoopbackRenderingDelayMicroseconds() const;
// Informs FilterGroups that the output should be mixed to mono.
void SetMixToMono(bool mix_to_mono);
// Informs FilterGroups and PostProcessors which channel will be played out.
// |playout_channel| may be |-1| to signal all channels will be played out.
void SetPlayoutChannel(int playout_channel);
......
......@@ -230,6 +230,7 @@ StreamMixer::StreamMixer(
CreatePostProcessors([](bool, const std::string&) {},
"" /* override_config */);
mixer_pipeline_->SetPlayoutChannel(playout_channel_);
// TODO(jyw): command line flag for filter frame alignment.
DCHECK_EQ(filter_frame_alignment_ & (filter_frame_alignment_ - 1), 0)
......@@ -580,14 +581,15 @@ void StreamMixer::UpdatePlayoutChannel() {
std::min(it.second->source()->playout_channel(), playout_channel);
}
}
if (playout_channel == playout_channel_) {
return;
}
DCHECK(playout_channel == kChannelAll ||
playout_channel >= 0 && playout_channel < kNumInputChannels);
LOG(INFO) << "Update playout channel: " << playout_channel;
mixer_pipeline_->SetMixToMono(num_output_channels_ == 1 &&
playout_channel == kChannelAll);
mixer_pipeline_->SetPlayoutChannel(playout_channel);
playout_channel_ = playout_channel;
mixer_pipeline_->SetPlayoutChannel(playout_channel_);
}
MediaPipelineBackend::AudioDecoder::RenderingDelay
......@@ -655,13 +657,7 @@ void StreamMixer::WriteMixedPcm(int frames, int64_t expected_playback_time) {
float* mixed_data = mixer_pipeline_->GetLoopbackOutput();
if (num_output_channels_ == 1 && mix_channel_count != 1) {
for (int i = 0; i < frames; ++i) {
float sum = 0;
for (int c = 0; c < mix_channel_count; ++c) {
sum += mixed_data[i * mix_channel_count + c];
}
mixed_data[i] = sum / mix_channel_count;
}
MixToMono(mixed_data, frames, mix_channel_count);
loopback_channel_count = 1;
}
......@@ -685,8 +681,14 @@ void StreamMixer::WriteMixedPcm(int frames, int64_t expected_playback_time) {
float* linearized_data = mixer_pipeline_->GetOutput();
int linearize_channel_count = mixer_pipeline_->GetOutputChannelCount();
if (num_output_channels_ == 1 && linearize_channel_count != 1) {
for (int i = 0; i < frames; ++i) {
linearized_data[i] = linearized_data[i * linearize_channel_count];
MixToMono(linearized_data, frames, linearize_channel_count);
} else if (num_output_channels_ > 1 && playout_channel_ != kChannelAll) {
// Duplicate selected channel to all channels.
for (int f = 0; f < frames; ++f) {
float selected =
linearized_data[f * num_output_channels_ + playout_channel_];
for (int c = 0; c < num_output_channels_; ++c)
linearized_data[f * num_output_channels_ + c] = selected;
}
}
......@@ -704,6 +706,23 @@ void StreamMixer::WriteMixedPcm(int frames, int64_t expected_playback_time) {
}
}
void StreamMixer::MixToMono(float* data, int frames, int channels) {
DCHECK_EQ(num_output_channels_, 1);
if (playout_channel_ == kChannelAll) {
for (int i = 0; i < frames; ++i) {
float sum = 0;
for (int c = 0; c < channels; ++c) {
sum += data[i * channels + c];
}
data[i] = sum / channels;
}
} else {
for (int i = 0; i < frames; ++i) {
data[i] = data[i * channels + playout_channel_];
}
}
}
void StreamMixer::AddLoopbackAudioObserver(
CastMediaShlib::LoopbackAudioObserver* observer) {
VLOG(1) << __func__;
......
......@@ -176,6 +176,7 @@ class StreamMixer {
void PlaybackLoop();
void WriteOneBuffer();
void WriteMixedPcm(int frames, int64_t expected_playback_time);
void MixToMono(float* data, int frames, int channels);
void SetVolumeOnThread(AudioContentType type, float level);
void SetMutedOnThread(AudioContentType type, bool muted);
......@@ -235,6 +236,7 @@ class StreamMixer {
// AudioPostProcessors require stricter alignment conditions.
const int filter_frame_alignment_;
int playout_channel_ = kChannelAll;
int requested_output_samples_per_second_ = 0;
int output_samples_per_second_ = 0;
int frames_per_write_ = 0;
......
......@@ -955,7 +955,7 @@ TEST_F(StreamMixerTest, PicksPlayoutChannel) {
VerifyAndClearPostProcessors(factory_ptr);
// Requests: all = 0 ch0 = 0 ch1 = 2.
EXPECT_CALL_ALL_POSTPROCESSORS(factory_ptr, UpdatePlayoutChannel(1));
// Playout channel is still 1.
mixer_->AddInput(&input4);
WaitForMixer();
VerifyAndClearPostProcessors(factory_ptr);
......@@ -969,8 +969,7 @@ TEST_F(StreamMixerTest, PicksPlayoutChannel) {
VerifyAndClearPostProcessors(factory_ptr);
// Requests: all = 1 ch0 = 0 ch1 = 1.
EXPECT_CALL_ALL_POSTPROCESSORS(factory_ptr,
UpdatePlayoutChannel(kChannelAll));
// Playout channel is still 'all'.
mixer_->RemoveInput(&input3);
WaitForMixer();
VerifyAndClearPostProcessors(factory_ptr);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment