Skip to content

GitLab

T
tangled
Commit 45a0c949 authored by Raymond Toy's avatar Raymond Toy Committed by Commit Bot
Browse files
Options

Implement tail processing for AudioNodes 

Enable tail processing by calling the necessary routines.  A
continuation of https://chromium-review.googlesource.com/c/chromium/src/+/949762
where tail processing is actually running.

Keep nodes alive when there are no input connections so that the node
has time to flush out any internal memory. When the output of the node
is going to be disabled (because there are no inputs), place the node
on a list, without disabling the output.

The list is processed every rendering quantum to see if the tail time
of node has passed.  If the tail time has not passed, nothing is done;
otherwise, the output is disabled, and the node is removed from the
list.  This allows the node to be collected, if possible.

Bug:357843, 731518
Test:AudioNode/tail-processing.html,DynamicsCompressor/dynamicscompressor-clear-internal-state.html

Change-Id: I5ba37015787fbbb2342eacb57b4851b99a36b3c5
Reviewed-on: https://chromium-review.googlesource.com/661165
Commit-Queue: Raymond Toy <rtoy@chromium.org>
Reviewed-by: default avatarHongchan Choi <hongchan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#543881}
parent 56d8b279
Hide whitespace changes
Inline Side-by-side
Showing with 415 additions and 51 deletions
third_party/WebKit/LayoutTests/TestExpectations
View file @ 45a0c949
......@@ -3385,6 +3385,3 @@ crbug.com/681919 [ Linux Debug ] http/tests/media/media-src-suspend-before-have-
# Sheriff 2018-03-07
crbug.com/819591 virtual/threaded/animations/hit-testing/composited-with-hit-testing.html [ Failure Pass ]
crbug.com/819778 [ Linux ] external/wpt/css/cssom-view/interfaces.html [ Pass Timeout ]
# Sheriff 2018-03-08
crbug.com/731518 [ Win Linux Mac ] webaudio/DynamicsCompressor/dynamicscompressor-clear-internal-state.html [ Failure Pass ]
third_party/WebKit/LayoutTests/webaudio/AudioNode/tail-processing.html 0 → 100644
View file @ 45a0c949
<!DOCTYPE html>
<html>
<head>
<title>
Test Handling of Tail Processing
</title>
<script src="../../resources/testharness.js"></script>
<script src="../../resources/testharnessreport.js"></script>
<script src="../resources/audit.js"></script>
<script src="../resources/audit-util.js"></script>
</head>
<body>
<script id="layout-test-code">
let audit = Audit.createTaskRunner();
// Fairly arbitrary but must be a power of two to eliminate roundoff when
// we compute times from sample frames
const sampleRate = 32768;
// Fairly arbitrary duration
const renderDuration = 0.25;
const renderFrames = renderDuration * sampleRate;
audit.define(
{label: 'grain', description: 'ABSN grain has the correct duration'},
(task, should) => {
// Test that the grain duration is the requested duration.
let context = new OfflineAudioContext(2, sampleRate, sampleRate);
let merger = new ChannelMergerNode(
context, {numberOfInputs: context.destination.channelCount});
merger.connect(context.destination);
// Number of frames for the grain we want to play. (Pretty much
// arbitrary.)
let grainFrames = 128;
// Test signal. Just fill the buffer with all ones.
let bTest = new AudioBuffer(
{length: context.length, sampleRate: context.sampleRate});
bTest.getChannelData(0).fill(1);
let sTest = new AudioBufferSourceNode(context, {buffer: bTest});
// Reference signal.
let bRef = new AudioBuffer(
{length: context.length, sampleRate: context.sampleRate});
bRef.getChannelData(0).fill(1, 0, grainFrames);
let sRef = new AudioBufferSourceNode(context, {buffer: bRef});
sTest.connect(merger, 0, 0);
sRef.connect(merger, 0, 1);
sTest.start(0, 0, grainFrames / context.sampleRate);
sRef.start();
context.startRendering()
.then(renderedBuffer => {
let actual = renderedBuffer.getChannelData(0);
let expected = renderedBuffer.getChannelData(1);
// The actual and expected should be the same because we got
// rid of the grain hack that caused a grain to play for an
// extra 512 frames.
should(actual, 'Rendered ABSN grain')
.beEqualToArray(expected);
})
.then(() => task.done());
});
audit.define('hrtf-panner-tail', (task, should) => {
runTest('PannerNode', {panningModel: 'HRTF', distanceMode: 'linear'})
.then(renderedBuffer => {
let prefix = 'HRTF PannerNode';
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let latencyFrame = findLatencyFrame(response);
let tailFrame = findTailFrame(response);
// The HRTF panner has both a latency component and a tail
// component. Make sure both are non-zero.
should(latencyFrame, `${prefix} latency frame (${latencyFrame})`)
.beGreaterThan(0);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
// Because of the latency, the output is zero at the beginning.
// Make sure this is true.
should(
output.slice(0, latencyFrame),
`${prefix} Latency output[0:` + (latencyFrame - 1) + ']')
.beConstantValueOf(0);
// Verify the rest of the output matches the expected values. The
// output should be non-zero from latencyFrame to tailFrame and
// zero after tailFrame.
verifyOutput(should, output, {
prefix: prefix,
startFrame: latencyFrame,
nonZeroEndFrame: Math.min(tailFrame, output.length),
zeroStartFrame: roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.define('biquad-tail', (task, should) => {
runTest('BiquadFilterNode', {Q: 20, frequency: 100})
.then(renderedBuffer => {
let prefix = 'BiquadFilter'
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let tailFrame = findTailFrame(response);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
// Verify biquad output which should be non-zero up to tailFrame
// and zero afterwards. However, the actual output isn't after
// tailFrame because the internal biquad tail time uses an
// approximation. That's why zeroStartFrame is one render quantum
// after tailFrame.
verifyOutput(should, output, {
prefix: prefix,
startFrame: 0,
nonZeroEndFrame:
Math.min(tailFrame + RENDER_QUANTUM_FRAMES, output.length),
zeroStartFrame: RENDER_QUANTUM_FRAMES + roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.define('iir-tail', (task, should) => {
runTest('IIRFilterNode', {feedforward: [1], feedback: [1, -.99]})
.then(renderedBuffer => {
let prefix = 'IIRFilter';
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let tailFrame = findTailFrame(response);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
verifyOutput(should, output, {
prefix: prefix,
startFrame: 0,
nonZeroEndFrame:
Math.min(tailFrame + RENDER_QUANTUM_FRAMES, output.length),
zeroStartFrame: 2 * RENDER_QUANTUM_FRAMES + roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.define('delay-tail', (task, should) => {
// For the test, make sure the delay is greater than one render
// quantum. If it's less we won't be able to tell if tail processing
// worked because the input signal is an impulse.
let delayFrames = RENDER_QUANTUM_FRAMES + 64;
runTest('DelayNode', {delayTime: delayFrames / sampleRate})
.then(renderedBuffer => {
let prefix = 'Delay';
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let tailFrame = findTailFrame(response);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
// As a delay node with delay time greater than one render
// quantum, the first render quantum must be 0.
should(
output.slice(0, RENDER_QUANTUM_FRAMES),
`${prefix} output[0:` + (RENDER_QUANTUM_FRAMES - 1) + ']')
.beConstantValueOf(0);
// The output of the delay node should be nonzero in the second
// render quantum and zero forever after.
verifyOutput(should, output, {
prefix: prefix,
startFrame: RENDER_QUANTUM_FRAMES,
nonZeroEndFrame: Math.min(tailFrame, output.length),
zeroStartFrame: roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.define('convolver-tail', (task, should) => {
// The convolver response. It needs to be longer than one render
// quantum to show the tail processing.
let response = new AudioBuffer(
{length: RENDER_QUANTUM_FRAMES + 64, sampleRate: sampleRate});
// For simplicity, just make the response all ones.
response.getChannelData(0).fill(1);
runTest('ConvolverNode', {disableNormalization: true, buffer: response})
.then(renderedBuffer => {
let prefix = 'Convolver';
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let tailFrame = findTailFrame(response);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
verifyOutput(should, output, {
prefix: prefix,
startFrame: 0,
nonZeroEndFrame:
Math.min(tailFrame + RENDER_QUANTUM_FRAMES, output.length),
zeroStartFrame: RENDER_QUANTUM_FRAMES + roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.define('dynamics-compressor-tail', (task, should) => {
runTest('DynamicsCompressorNode', {})
.then(renderedBuffer => {
let prefix = 'DyamicsCompressor';
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let tailFrame = findTailFrame(response);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
let latencyFrame = roundDown(tailFrame - 1);
should(
output.slice(0, latencyFrame),
`${prefix} output[0:` + (latencyFrame - 1) + ']')
.beConstantValueOf(0);
verifyOutput(should, output, {
prefix: prefix,
startFrame: latencyFrame,
nonZeroEndFrame: Math.min(tailFrame, output.length),
zeroStartFrame: roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.define('waveshaper-tail', (task, should) => {
// Fairly arbitrary curve for the WaveShaper
let curve = Float32Array.from([-1, -.5, 0, 0.5, 1]);
runTest('WaveShaperNode', {curve: curve, oversample: '2x'})
.then(renderedBuffer => {
let prefix = 'WaveShaper';
let output = renderedBuffer.getChannelData(0);
let response = renderedBuffer.getChannelData(1);
let tailFrame = findTailFrame(response);
should(tailFrame, `${prefix} tail frame (${tailFrame})`)
.beGreaterThan(0);
verifyOutput(should, output, {
prefix: prefix,
startFrame: 0,
nonZeroEndFrame: Math.min(tailFrame, output.length),
zeroStartFrame: roundUp(tailFrame),
tailFrame: tailFrame,
reference: response
});
})
.then(() => task.done());
});
audit.run();
function runTest(nodeName, nodeOptions) {
// Two-channel output. Channel 0 is the test result; channel 1 is the
// impulse response that is used to figure out when the tail should
// start.
let context = new OfflineAudioContext(2, sampleRate, sampleRate);
// Merge channels for the destination.
let merger = new ChannelMergerNode(context, {numberOfInputs: 2});
merger.connect(context.destination);
let src = new ConstantSourceNode(context, {offset: 1});
// Impulse for testing. We want a full buffer so as not to worry about
// the source disconnecting prematurely from the filter.
let b = new AudioBuffer(
{length: context.length, sampleRate: context.sampleRate});
b.getChannelData(0)[0] = 1;
let impulse = new AudioBufferSourceNode(context, {buffer: b});
let testNode = new window[nodeName](context, nodeOptions);
let refNode = new window[nodeName](context, nodeOptions);
src.connect(testNode).connect(merger, 0, 0);
impulse.connect(refNode).connect(merger, 0, 1);
src.start();
src.stop(1 / context.sampleRate);
impulse.start();
return context.startRendering();
}
// Starting from the end find the first frame that exceeds our threshold.
// This assumes that everything less than the threshold is equivalent to 0
// for our purposes.
function findTailFrame(response) {
// Any value below this is considered to be zero for our purpose of
// finding the first non-zero value. Somewhat arbitrary, but the value
// here is the size of the LSB of a 16-bit PCM sample.
let zeroThreshold = 1 / 32768.
let tailFrame = response.length;
for (let k = response.length - 1; k >= 0; --k) {
if (Math.abs(response[k]) > zeroThreshold) {
tailFrame = k + 1;
break;
}
}
return tailFrame;
}
function findLatencyFrame(response) {
for (let k = 0; k < response.length; ++k) {
if (response[k] != 0)
return k;
}
return response.length;
}
function verifyOutput(should, output, options) {
let prefix = options.prefix || '';
if (options.tailFrame && options.reference) {
should(
output.slice(0, options.tailFrame),
`${prefix} Tail output[0:` + (options.tailFrame - 1) + ']')
.beEqualToArray(options.reference.slice(0, options.tailFrame));
}
// Verify that |output| is non-zero between |startFrame| and
// |nonZeroEndFrame|.
for (let k = options.startFrame; k < options.nonZeroEndFrame;
k += RENDER_QUANTUM_FRAMES) {
should(
output.slice(k, k + RENDER_QUANTUM_FRAMES),
`${prefix} output[` + k + ':' + (k + 127) + ']')
.notBeConstantValueOf(0);
}
// Verify |output| is zero starting at frame |zeroStartFrame|, inclusive
if (options.zeroStartFrame < output.length) {
should(
output.slice(options.zeroStartFrame),
`${prefix} output[` + options.zeroStartFrame + ':]')
.beConstantValueOf(0);
}
}
function roundDown(frame) {
return RENDER_QUANTUM_FRAMES *
Math.floor(frame / RENDER_QUANTUM_FRAMES);
}
function roundUp(frame) {
return RENDER_QUANTUM_FRAMES * Math.ceil(frame / RENDER_QUANTUM_FRAMES);
}
</script>
</body>
</html>
third_party/WebKit/LayoutTests/webaudio/DynamicsCompressor/dynamicscompressor-clear-internal-state.html
View file @ 45a0c949
......@@ -53,7 +53,8 @@
let errorThreshold = 4.8223e-2;
should(
Math.abs(compressor.reduction),
'Math.abs(compressor.reduction)')
'Math.abs(compressor.reduction) ('
+ Math.abs(compressor.reduction) + ')')
.beLessThanOrEqualTo(errorThreshold);
task.done();
});
......
third_party/WebKit/Source/modules/webaudio/AudioNode.cpp
View file @ 45a0c949
......@@ -103,8 +103,6 @@ void AudioHandler::Uninitialize() {
is_initialized_ = false;
}
void AudioHandler::ClearInternalStateWhenDisabled() {}
void AudioHandler::Dispose() {
DCHECK(IsMainThread());
DCHECK(Context()->IsGraphOwner());
......@@ -338,14 +336,6 @@ void AudioHandler::ProcessIfNecessary(size_t frames_to_process) {
PullInputs(frames_to_process);
bool silent_inputs = InputsAreSilent();
if (!silent_inputs) {
// Update |last_non_silent_time| AFTER processing this block.
// Doing it before causes |PropagateSilence()| to be one render
// quantum longer than necessary.
last_non_silent_time_ =
(Context()->CurrentSampleFrame() + frames_to_process) /
static_cast<double>(Context()->sampleRate());
}
if (silent_inputs && PropagatesSilence()) {
SilenceOutputs();
// AudioParams still need to be processed so that the value can be updated
......@@ -360,6 +350,15 @@ void AudioHandler::ProcessIfNecessary(size_t frames_to_process) {
UnsilenceOutputs();
Process(frames_to_process);
}
if (!silent_inputs) {
// Update |last_non_silent_time| AFTER processing this block.
// Doing it before causes |PropagateSilence()| to be one render
// quantum longer than necessary.
last_non_silent_time_ =
(Context()->CurrentSampleFrame() + frames_to_process) /
static_cast<double>(Context()->sampleRate());
}
}
}
......@@ -437,25 +436,16 @@ void AudioHandler::DisableOutputsIfNecessary() {
// they're connected to. disable() can recursively deref connections (and
// call disable()) down a whole chain of connected nodes.
// TODO(rtoy,hongchan): we need special cases the convolver, delay, biquad,
// and IIR since they have a significant tail-time and shouldn't be
// disconnected simply because they no longer have any input connections.
// This needs to be handled more generally where AudioNodes have a tailTime
// attribute. Then the AudioNode only needs to remain "active" for tailTime
// seconds after there are no longer any active connections.
//
// The analyser node also requires special handling because we
// need the internal state to be updated for the time and FFT data
// even if it has no connections.
if (GetNodeType() != kNodeTypeConvolver &&
GetNodeType() != kNodeTypeDelay &&
GetNodeType() != kNodeTypeBiquadFilter &&
GetNodeType() != kNodeTypeIIRFilter &&
GetNodeType() != kNodeTypeAnalyser) {
is_disabled_ = true;
ClearInternalStateWhenDisabled();
for (auto& output : outputs_)
output->Disable();
// If a node requires tail processing, we defer the disabling of
// the outputs so that the tail for the node can be output.
// Otherwise, we can disable the outputs right away.
if (RequiresTailProcessing()) {
if (Context()->ContextState() !=
BaseAudioContext::AudioContextState::kClosed) {
Context()->GetDeferredTaskHandler().AddTailProcessingHandler(this);
}
} else {
DisableOutputs();
}
}
}
......@@ -469,11 +459,14 @@ void AudioHandler::DisableOutputs() {
void AudioHandler::MakeConnection() {
AtomicIncrement(&connection_ref_count_);
Context()->GetDeferredTaskHandler().RemoveTailProcessingHandler(this);
#if DEBUG_AUDIONODE_REFERENCES
fprintf(stderr,
"[%16p]: %16p: %2d: AudioHandler::MakeConnection %3d [%3d]\n",
Context(), this, GetNodeType(), connection_ref_count_,
node_count_[GetNodeType()]);
fprintf(
stderr,
"[%16p]: %16p: %2d: AudioHandler::MakeConnection %3d [%3d] @%.15g\n",
Context(), this, GetNodeType(), connection_ref_count_,
node_count_[GetNodeType()], Context()->currentTime());
#endif
// See the disabling code in disableOutputsIfNecessary(). This handles
// the case where a node is being re-connected after being used at least
......
third_party/WebKit/Source/modules/webaudio/AudioNode.h
View file @ 45a0c949
......@@ -152,14 +152,6 @@ class MODULES_EXPORT AudioHandler : public ThreadSafeRefCounted<AudioHandler> {
virtual void Initialize();
virtual void Uninitialize();
// Clear internal state when the node is disabled. When a node is disabled,
// it is no longer pulled so any internal state is never updated. But some
// nodes (DynamicsCompressorNode) have internal state that is still
// accessible by the user. Update the internal state as if the node were
// still connected but processing all zeroes. This gives a consistent view
// to the user.
virtual void ClearInternalStateWhenDisabled();
bool IsInitialized() const { return is_initialized_; }
unsigned NumberOfInputs() const { return inputs_.size(); }
......
third_party/WebKit/Source/modules/webaudio/DynamicsCompressorNode.cpp
View file @ 45a0c949
......@@ -122,10 +122,6 @@ void DynamicsCompressorHandler::Initialize() {
Context()->sampleRate(), defaultNumberOfOutputChannels);
}
void DynamicsCompressorHandler::ClearInternalStateWhenDisabled() {
reduction_ = 0;
}
bool DynamicsCompressorHandler::RequiresTailProcessing() const {
// Always return true even if the tail time and latency might both be zero.
return true;
......
third_party/WebKit/Source/modules/webaudio/DynamicsCompressorNode.h
View file @ 45a0c949
......@@ -55,7 +55,6 @@ class MODULES_EXPORT DynamicsCompressorHandler final : public AudioHandler {
void Process(size_t frames_to_process) override;
void ProcessOnlyAudioParams(size_t frames_to_process) override;
void Initialize() override;
void ClearInternalStateWhenDisabled() override;
float ReductionValue() const { return reduction_; }
......
third_party/WebKit/Source/modules/webaudio/PannerNode.cpp
View file @ 45a0c949
......@@ -102,7 +102,7 @@ PannerHandler::~PannerHandler() {
void PannerHandler::Process(size_t frames_to_process) {
AudioBus* destination = Output(0).Bus();
if (!IsInitialized() || !Input(0).IsConnected() || !panner_.get()) {
if (!IsInitialized() || !panner_.get()) {
destination->Zero();
return;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment