Commit 4234d1eb authored by Markus Handell's avatar Markus Handell Committed by Commit Bot

WebmMuxer: Do not produce duplicate timestamps.

This change fixes a problem where the WebmMuxer will
produce duplicate timestamps leading to audio gaps or
jerky video in WebM files, provided both audio and
video is recorded, and timestamps of all incoming
frames are not monotonically increasing. This
resulted in the WebmMuxer rewriting apparently older
frames' timestamps as the newest one encountered.

With this change we now normally don't change the incoming
timestamps, but always output a sorted sequence of frames
to the underlying Matroska muxer. Timestamps are
exceptionally changed when it's found that each stream's
timestamps aren't monotonically increasing.

Additionally, the webm_muxer got it's unit tests updated.

Bug: 873963
Change-Id: I6b81ef32f0f7f1034a56da3bca07186221fefaea
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2033253
Commit-Queue: Markus Handell <handellm@google.com>
Reviewed-by: default avatarMiguel Casas <mcasas@chromium.org>
Reviewed-by: default avatarDale Curtis <dalecurtis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#738738}
parent 0d8a0435
...@@ -192,6 +192,7 @@ WebmMuxer::~WebmMuxer() { ...@@ -192,6 +192,7 @@ WebmMuxer::~WebmMuxer() {
// No need to segment_.Finalize() since is not Seekable(), i.e. a live // No need to segment_.Finalize() since is not Seekable(), i.e. a live
// stream, but is a good practice. // stream, but is a good practice.
DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(thread_checker_.CalledOnValidThread());
FlushQueues();
segment_.Finalize(); segment_.Finalize();
} }
...@@ -228,19 +229,15 @@ bool WebmMuxer::OnEncodedVideo(const VideoParameters& params, ...@@ -228,19 +229,15 @@ bool WebmMuxer::OnEncodedVideo(const VideoParameters& params,
if (has_audio_ && !audio_track_index_) { if (has_audio_ && !audio_track_index_) {
DVLOG(1) << __func__ << ": delaying until audio track ready."; DVLOG(1) << __func__ << ": delaying until audio track ready.";
if (is_key_frame) // Upon Key frame reception, empty the encoded queue. if (is_key_frame) // Upon Key frame reception, empty the encoded queue.
encoded_frames_queue_.clear(); video_frames_.clear();
encoded_frames_queue_.push_back(std::make_unique<EncodedVideoFrame>(
std::move(encoded_data), std::move(encoded_alpha), timestamp,
is_key_frame));
return true;
} }
// Any saved encoded video frames must have been dumped in OnEncodedAudio(); const base::TimeTicks recorded_timestamp =
DCHECK(encoded_frames_queue_.empty()); UpdateLastTimestampMonotonically(timestamp, &last_frame_timestamp_video_);
video_frames_.push_back(EncodedFrame{
return AddFrame(encoded_data, encoded_alpha, video_track_index_, std::move(encoded_data), std::move(encoded_alpha),
timestamp - first_frame_timestamp_video_, is_key_frame); recorded_timestamp - first_frame_timestamp_video_, is_key_frame});
return PartiallyFlushQueues();
} }
bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params, bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
...@@ -253,29 +250,16 @@ bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params, ...@@ -253,29 +250,16 @@ bool WebmMuxer::OnEncodedAudio(const media::AudioParameters& params,
AddAudioTrack(params); AddAudioTrack(params);
if (first_frame_timestamp_audio_.is_null()) if (first_frame_timestamp_audio_.is_null())
first_frame_timestamp_audio_ = timestamp; first_frame_timestamp_audio_ = timestamp;
last_frame_timestamp_video_ = timestamp;
} }
// TODO(ajose): Don't drop audio data: http://crbug.com/547948 const base::TimeTicks recorded_timestamp =
// TODO(ajose): Support multiple tracks: http://crbug.com/528523 UpdateLastTimestampMonotonically(timestamp, &last_frame_timestamp_audio_);
if (has_video_ && !video_track_index_) { audio_frames_.push_back(
DVLOG(1) << __func__ << ": delaying until video track ready."; EncodedFrame{encoded_data, std::string(),
return true; recorded_timestamp - first_frame_timestamp_audio_,
} /*is_keyframe=*/true});
return PartiallyFlushQueues();
// Dump all saved encoded video frames if any.
while (!encoded_frames_queue_.empty()) {
const bool res = AddFrame(
encoded_frames_queue_.front()->data,
encoded_frames_queue_.front()->alpha_data, video_track_index_,
encoded_frames_queue_.front()->timestamp - first_frame_timestamp_video_,
encoded_frames_queue_.front()->is_keyframe);
if (!res)
return false;
encoded_frames_queue_.pop_front();
}
return AddFrame(encoded_data, std::string(), audio_track_index_,
timestamp - first_frame_timestamp_audio_,
true /* is_key_frame -- always true for audio */);
} }
void WebmMuxer::Pause() { void WebmMuxer::Pause() {
...@@ -412,17 +396,41 @@ void WebmMuxer::ElementStartNotify(mkvmuxer::uint64 element_id, ...@@ -412,17 +396,41 @@ void WebmMuxer::ElementStartNotify(mkvmuxer::uint64 element_id,
<< "Can't go back in a live WebM stream."; << "Can't go back in a live WebM stream.";
} }
bool WebmMuxer::AddFrame(const std::string& encoded_data, void WebmMuxer::FlushQueues() {
const std::string& encoded_alpha,
uint8_t track_index,
base::TimeDelta timestamp,
bool is_key_frame) {
DCHECK(thread_checker_.CalledOnValidThread()); DCHECK(thread_checker_.CalledOnValidThread());
DCHECK(!has_video_ || video_track_index_); while ((!video_frames_.empty() || !audio_frames_.empty()) &&
DCHECK(!has_audio_ || audio_track_index_); FlushNextFrame()) {
}
}
most_recent_timestamp_ = bool WebmMuxer::PartiallyFlushQueues() {
std::max(most_recent_timestamp_, timestamp - total_time_in_pause_); DCHECK(thread_checker_.CalledOnValidThread());
bool result = true;
while (!(has_video_ && video_frames_.empty()) &&
!(has_audio_ && audio_frames_.empty()) && result) {
result = FlushNextFrame();
}
return result;
}
bool WebmMuxer::FlushNextFrame() {
DCHECK(thread_checker_.CalledOnValidThread());
base::TimeDelta min_timestamp = base::TimeDelta::Max();
base::circular_deque<EncodedFrame>* queue = &video_frames_;
uint8_t track_index = video_track_index_;
if (!video_frames_.empty())
min_timestamp = video_frames_.front().relative_timestamp;
if (!audio_frames_.empty() &&
audio_frames_.front().relative_timestamp < min_timestamp) {
queue = &audio_frames_;
track_index = audio_track_index_;
}
EncodedFrame frame = std::move(queue->front());
queue->pop_front();
auto recorded_timestamp = frame.relative_timestamp.InMicroseconds() *
base::Time::kNanosecondsPerMicrosecond;
if (force_one_libwebm_error_) { if (force_one_libwebm_error_) {
DVLOG(1) << "Forcing a libwebm error"; DVLOG(1) << "Forcing a libwebm error";
...@@ -430,36 +438,32 @@ bool WebmMuxer::AddFrame(const std::string& encoded_data, ...@@ -430,36 +438,32 @@ bool WebmMuxer::AddFrame(const std::string& encoded_data,
return false; return false;
} }
DCHECK(encoded_data.data()); DCHECK(frame.data.data());
if (encoded_alpha.empty()) { bool result =
return segment_.AddFrame( frame.alpha_data.empty()
reinterpret_cast<const uint8_t*>(encoded_data.data()), ? segment_.AddFrame(
encoded_data.size(), track_index, reinterpret_cast<const uint8_t*>(frame.data.data()),
most_recent_timestamp_.InMicroseconds() * frame.data.size(), track_index, recorded_timestamp,
base::Time::kNanosecondsPerMicrosecond, frame.is_keyframe)
is_key_frame); : segment_.AddFrameWithAdditional(
} reinterpret_cast<const uint8_t*>(frame.data.data()),
frame.data.size(),
DCHECK(encoded_alpha.data()); reinterpret_cast<const uint8_t*>(frame.alpha_data.data()),
return segment_.AddFrameWithAdditional( frame.alpha_data.size(), 1 /* add_id */, track_index,
reinterpret_cast<const uint8_t*>(encoded_data.data()), recorded_timestamp, frame.is_keyframe);
encoded_data.size(), return result;
reinterpret_cast<const uint8_t*>(encoded_alpha.data()),
encoded_alpha.size(), 1 /* add_id */, track_index,
most_recent_timestamp_.InMicroseconds() *
base::Time::kNanosecondsPerMicrosecond,
is_key_frame);
} }
WebmMuxer::EncodedVideoFrame::EncodedVideoFrame(std::string data, base::TimeTicks WebmMuxer::UpdateLastTimestampMonotonically(
std::string alpha_data, base::TimeTicks timestamp,
base::TimeTicks timestamp, base::TimeTicks* last_timestamp) {
bool is_keyframe) // In theory, time increases monotonically. In practice, it does not.
: data(std::move(data)), // See http://crbug/618407.
alpha_data(std::move(alpha_data)), DLOG_IF(WARNING, timestamp < *last_timestamp)
timestamp(timestamp), << "Encountered a non-monotonically increasing timestamp. Was: "
is_keyframe(is_keyframe) {} << *last_timestamp << ", now: " << timestamp;
*last_timestamp = std::max(*last_timestamp, timestamp);
WebmMuxer::EncodedVideoFrame::~EncodedVideoFrame() = default; return *last_timestamp;
}
} // namespace media } // namespace media
...@@ -112,12 +112,30 @@ class MEDIA_EXPORT WebmMuxer : public mkvmuxer::IMkvWriter { ...@@ -112,12 +112,30 @@ class MEDIA_EXPORT WebmMuxer : public mkvmuxer::IMkvWriter {
void ElementStartNotify(mkvmuxer::uint64 element_id, void ElementStartNotify(mkvmuxer::uint64 element_id,
mkvmuxer::int64 position) override; mkvmuxer::int64 position) override;
// Helper to simplify saving frames. Returns true on success. // Adds all currently buffered frames to the mkvmuxer in timestamp order,
bool AddFrame(const std::string& encoded_data, // until the queues are depleted.
const std::string& encoded_alpha_data, void FlushQueues();
uint8_t track_index, // Flushes out frames to the mkvmuxer while ensuring monotonically increasing
base::TimeDelta timestamp, // timestamps as per the WebM specification,
bool is_key_frame); // https://www.webmproject.org/docs/container/. Returns true on success and
// false on mkvmuxer failure.
//
// Note that frames may still be around in the queues after this call. The
// method stops flushing when timestamp monotonicity can't be guaranteed
// anymore.
bool PartiallyFlushQueues();
// Flushes out the next frame in timestamp order from the queues. Returns true
// on success and false on mkvmuxer failure.
//
// Note: it's assumed that at least one video or audio frame is queued.
bool FlushNextFrame();
// Calculates a monotonically increasing timestamp from an input |timestamp|
// and a pointer to a previously stored |last_timestamp| by taking the maximum
// of |timestamp| and *|last_timestamp|. Updates *|last_timestamp| if
// |timestamp| is greater.
base::TimeTicks UpdateLastTimestampMonotonically(
base::TimeTicks timestamp,
base::TimeTicks* last_timestamp);
// Used to DCHECK that we are called on the correct thread. // Used to DCHECK that we are called on the correct thread.
base::ThreadChecker thread_checker_; base::ThreadChecker thread_checker_;
...@@ -134,8 +152,9 @@ class MEDIA_EXPORT WebmMuxer : public mkvmuxer::IMkvWriter { ...@@ -134,8 +152,9 @@ class MEDIA_EXPORT WebmMuxer : public mkvmuxer::IMkvWriter {
// Origin of times for frame timestamps. // Origin of times for frame timestamps.
base::TimeTicks first_frame_timestamp_video_; base::TimeTicks first_frame_timestamp_video_;
base::TimeTicks last_frame_timestamp_video_;
base::TimeTicks first_frame_timestamp_audio_; base::TimeTicks first_frame_timestamp_audio_;
base::TimeDelta most_recent_timestamp_; base::TimeTicks last_frame_timestamp_audio_;
// Variables to measure and accumulate, respectively, the time in pause state. // Variables to measure and accumulate, respectively, the time in pause state.
std::unique_ptr<base::ElapsedTimer> elapsed_time_in_pause_; std::unique_ptr<base::ElapsedTimer> elapsed_time_in_pause_;
...@@ -157,25 +176,21 @@ class MEDIA_EXPORT WebmMuxer : public mkvmuxer::IMkvWriter { ...@@ -157,25 +176,21 @@ class MEDIA_EXPORT WebmMuxer : public mkvmuxer::IMkvWriter {
// Flag to force the next call to a |segment_| method to return false. // Flag to force the next call to a |segment_| method to return false.
bool force_one_libwebm_error_; bool force_one_libwebm_error_;
// Hold on to all encoded video frames to dump them with and when audio is struct EncodedFrame {
// received, if expected, since WebM headers can only be written once.
struct EncodedVideoFrame {
EncodedVideoFrame(std::string data,
std::string alpha_data,
base::TimeTicks timestamp,
bool is_keyframe);
~EncodedVideoFrame();
std::string data; std::string data;
std::string alpha_data; std::string alpha_data;
base::TimeTicks timestamp; base::TimeDelta
relative_timestamp; // relative to first_frame_timestamp_xxx_
bool is_keyframe; bool is_keyframe;
private:
DISALLOW_IMPLICIT_CONSTRUCTORS(EncodedVideoFrame);
}; };
base::circular_deque<std::unique_ptr<EncodedVideoFrame>>
encoded_frames_queue_; // The following two queues hold frames to ensure that monotonically
// increasing timestamps are stored in the resulting webm file without
// modifying the timestamps.
base::circular_deque<EncodedFrame> audio_frames_;
// If muxing audio and video, this queue holds frames until the first audio
// frame appears.
base::circular_deque<EncodedFrame> video_frames_;
DISALLOW_COPY_AND_ASSIGN(WebmMuxer); DISALLOW_COPY_AND_ASSIGN(WebmMuxer);
}; };
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment