Commit eba9de37 authored by hubbe@chromium.org's avatar hubbe@chromium.org

Implement adaptive target delay extension

Adds new call on cast_sender: SetTargetPlayoutDelay
Caller is responsible for negotiating with the receiver if the extension is actually supported or not.

BUG=405339

Review URL: https://codereview.chromium.org/493823002

Cr-Commit-Position: refs/heads/master@{#291453}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@291453 0039d316-1c4b-4281-b951-d872f2087c98
parent 61dd4c07
......@@ -34,6 +34,7 @@ IPC_STRUCT_TRAITS_BEGIN(media::cast::EncodedFrame)
IPC_STRUCT_TRAITS_MEMBER(referenced_frame_id)
IPC_STRUCT_TRAITS_MEMBER(rtp_timestamp)
IPC_STRUCT_TRAITS_MEMBER(reference_time)
IPC_STRUCT_TRAITS_MEMBER(new_playout_delay_ms)
IPC_STRUCT_TRAITS_MEMBER(data)
IPC_STRUCT_TRAITS_END()
......
......@@ -86,6 +86,11 @@ class CastSender {
const CastInitializationCallback& cast_initialization_cb,
const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb) = 0;
// Change the target delay. This is only valid if the receiver
// supports the "adaptive_target_delay" rtp extension.
virtual void SetTargetPlayoutDelay(
base::TimeDelta new_target_playout_delay) = 0;
};
} // namespace cast
......
......@@ -110,6 +110,10 @@ void CastSenderImpl::InitializeAudio(
new LocalAudioFrameInput(cast_environment_, audio_sender_->AsWeakPtr());
}
cast_initialization_cb.Run(status);
if (video_sender_) {
DCHECK(audio_sender_->GetTargetPlayoutDelay() ==
video_sender_->GetTargetPlayoutDelay());
}
}
void CastSenderImpl::InitializeVideo(
......@@ -136,6 +140,10 @@ void CastSenderImpl::InitializeVideo(
new LocalVideoFrameInput(cast_environment_, video_sender_->AsWeakPtr());
}
cast_initialization_cb.Run(status);
if (audio_sender_) {
DCHECK(audio_sender_->GetTargetPlayoutDelay() ==
video_sender_->GetTargetPlayoutDelay());
}
}
CastSenderImpl::~CastSenderImpl() {
......@@ -150,5 +158,17 @@ scoped_refptr<VideoFrameInput> CastSenderImpl::video_frame_input() {
return video_frame_input_;
}
void CastSenderImpl::SetTargetPlayoutDelay(
base::TimeDelta new_target_playout_delay) {
VLOG(1) << "CastSenderImpl@" << this << "::SetTargetPlayoutDelay("
<< new_target_playout_delay.InMilliseconds() << " ms)";
if (audio_sender_) {
audio_sender_->SetTargetPlayoutDelay(new_target_playout_delay);
}
if (video_sender_) {
video_sender_->SetTargetPlayoutDelay(new_target_playout_delay);
}
}
} // namespace cast
} // namespace media
......@@ -37,6 +37,9 @@ class CastSenderImpl : public CastSender {
const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb)
OVERRIDE;
virtual void SetTargetPlayoutDelay(
base::TimeDelta new_target_playout_delay) OVERRIDE;
virtual ~CastSenderImpl();
virtual scoped_refptr<AudioFrameInput> audio_frame_input() OVERRIDE;
......
......@@ -16,7 +16,8 @@ EncodedFrame::EncodedFrame()
: dependency(UNKNOWN_DEPENDENCY),
frame_id(0),
referenced_frame_id(0),
rtp_timestamp(0) {}
rtp_timestamp(0),
new_playout_delay_ms(0) {}
EncodedFrame::~EncodedFrame() {}
......
......@@ -117,6 +117,10 @@ struct EncodedFrame {
// timestamps; and it may not necessarily increment with precise regularity.
base::TimeTicks reference_time;
// Playout delay for this and all future frames. Used by the Adaptive
// Playout delay extension. Zero means no change.
uint16 new_playout_delay_ms;
// The encoded signal data.
std::string data;
};
......
......@@ -13,6 +13,7 @@ FrameBuffer::FrameBuffer()
: frame_id_(0),
max_packet_id_(0),
num_packets_received_(0),
new_playout_delay_ms_(0),
is_key_frame_(false),
total_data_size_(0),
last_referenced_frame_id_(0),
......@@ -28,6 +29,7 @@ void FrameBuffer::InsertPacket(const uint8* payload_data,
frame_id_ = rtp_header.frame_id;
max_packet_id_ = rtp_header.max_packet_id;
is_key_frame_ = rtp_header.is_key_frame;
new_playout_delay_ms_ = rtp_header.new_playout_delay_ms;
if (is_key_frame_)
DCHECK_EQ(rtp_header.frame_id, rtp_header.reference_frame_id);
last_referenced_frame_id_ = rtp_header.reference_frame_id;
......@@ -73,6 +75,7 @@ bool FrameBuffer::AssembleEncodedFrame(EncodedFrame* frame) const {
frame->frame_id = frame_id_;
frame->referenced_frame_id = last_referenced_frame_id_;
frame->rtp_timestamp = rtp_timestamp_;
frame->new_playout_delay_ms = new_playout_delay_ms_;
// Build the data vector.
frame->data.clear();
......
......@@ -39,6 +39,7 @@ class FrameBuffer {
uint32 frame_id_;
uint16 max_packet_id_;
uint16 num_packets_received_;
uint16 new_playout_delay_ms_;
bool is_key_frame_;
size_t total_data_size_;
uint32 last_referenced_frame_id_;
......
// Copyright 2014 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
namespace media {
namespace cast {
static const uint16 kRtpHeaderLength = 12;
static const uint16 kCastHeaderLength = 7;
static const uint8 kRtpExtensionBitMask = 0x10;
static const uint8 kCastKeyFrameBitMask = 0x80;
static const uint8 kCastReferenceFrameIdBitMask = 0x40;
static const uint8 kRtpMarkerBitMask = 0x80;
static const uint8 kCastExtensionCountmask = 0x3f;
// Cast RTP extensions.
static const uint8 kCastRtpExtensionAdaptiveLatency = 1;
} // namespace cast
} // namespace media
......@@ -7,16 +7,11 @@
#include "base/big_endian.h"
#include "base/logging.h"
#include "media/cast/net/pacing/paced_sender.h"
#include "media/cast/net/rtp/rtp_defines.h"
namespace media {
namespace cast {
static const uint16 kCommonRtpHeaderLength = 12;
static const uint16 kCastRtpHeaderLength = 7;
static const uint8 kCastKeyFrameBitMask = 0x80;
static const uint8 kCastReferenceFrameIdBitMask = 0x40;
static const uint8 kRtpMarkerBitMask = 0x80;
RtpPacketizerConfig::RtpPacketizerConfig()
: payload_type(-1),
max_payload_length(kMaxIpPacketSize - 28), // Default is IP-v4/UDP.
......@@ -47,7 +42,7 @@ uint16 RtpPacketizer::NextSequenceNumber() {
}
void RtpPacketizer::SendFrameAsPackets(const EncodedFrame& frame) {
uint16 rtp_header_length = kCommonRtpHeaderLength + kCastRtpHeaderLength;
uint16 rtp_header_length = kRtpHeaderLength + kCastHeaderLength;
uint16 max_length = config_.max_payload_length - rtp_header_length - 1;
rtp_timestamp_ = frame.rtp_timestamp;
......@@ -73,9 +68,15 @@ void RtpPacketizer::SendFrameAsPackets(const EncodedFrame& frame) {
// Build Cast header.
// TODO(miu): Should we always set the ref frame bit and the ref_frame_id?
DCHECK_NE(frame.dependency, EncodedFrame::UNKNOWN_DEPENDENCY);
packet->data.push_back(
((frame.dependency == EncodedFrame::KEY) ? kCastKeyFrameBitMask : 0) |
kCastReferenceFrameIdBitMask);
uint8 num_extensions = 0;
if (frame.new_playout_delay_ms)
num_extensions++;
uint8 byte0 = kCastReferenceFrameIdBitMask;
if (frame.dependency == EncodedFrame::KEY)
byte0 |= kCastKeyFrameBitMask;
DCHECK_LE(num_extensions, kCastExtensionCountmask);
byte0 |= num_extensions;
packet->data.push_back(byte0);
packet->data.push_back(static_cast<uint8>(frame.frame_id));
size_t start_size = packet->data.size();
packet->data.resize(start_size + 4);
......@@ -84,6 +85,14 @@ void RtpPacketizer::SendFrameAsPackets(const EncodedFrame& frame) {
big_endian_writer.WriteU16(packet_id_);
big_endian_writer.WriteU16(static_cast<uint16>(num_packets - 1));
packet->data.push_back(static_cast<uint8>(frame.referenced_frame_id));
if (frame.new_playout_delay_ms) {
packet->data.push_back(kCastRtpExtensionAdaptiveLatency << 2);
packet->data.push_back(2); // 2 bytes
packet->data.push_back(
static_cast<uint8>(frame.new_playout_delay_ms >> 8));
packet->data.push_back(
static_cast<uint8>(frame.new_playout_delay_ms));
}
// Copy payload data.
packet->data.insert(packet->data.end(),
......
......@@ -7,17 +7,11 @@
#include "base/big_endian.h"
#include "base/logging.h"
#include "media/cast/cast_defines.h"
#include "media/cast/net/rtp/rtp_defines.h"
namespace media {
namespace cast {
static const size_t kRtpHeaderLength = 12;
static const size_t kCastHeaderLength = 7;
static const uint8 kRtpExtensionBitMask = 0x10;
static const uint8 kRtpMarkerBitMask = 0x80;
static const uint8 kCastKeyFrameBitMask = 0x80;
static const uint8 kCastReferenceFrameIdBitMask = 0x40;
RtpParser::RtpParser(uint32 expected_sender_ssrc, uint8 expected_payload_type)
: expected_sender_ssrc_(expected_sender_ssrc),
expected_payload_type_(expected_payload_type) {}
......@@ -92,6 +86,22 @@ bool RtpParser::ParsePacket(const uint8* packet,
return false;
}
for (int i = 0; i < (bits & kCastExtensionCountmask); i++) {
uint16 type_and_size;
if (!reader.ReadU16(&type_and_size))
return false;
base::StringPiece tmp;
if (!reader.ReadPiece(&tmp, type_and_size & 0x3ff))
return false;
base::BigEndianReader chunk(tmp.data(), tmp.size());
switch (type_and_size >> 10) {
case kCastRtpExtensionAdaptiveLatency:
if (!chunk.ReadU16(&header->new_playout_delay_ms))
return false;
}
}
// Only the lower 8 bits of the |frame_id| were serialized, so do some magic
// to restore the upper 24 bits.
//
......
......@@ -17,7 +17,8 @@ RtpCastHeader::RtpCastHeader()
frame_id(0),
packet_id(0),
max_packet_id(0),
reference_frame_id(0) {}
reference_frame_id(0),
new_playout_delay_ms(0) {}
RtpPayloadFeedback::~RtpPayloadFeedback() {}
......
......@@ -28,6 +28,8 @@ struct RtpCastHeader {
uint16 packet_id;
uint16 max_packet_id;
uint32 reference_frame_id;
uint16 new_playout_delay_ms;
};
class RtpPayloadFeedback {
......
......@@ -202,8 +202,7 @@ void FrameReceiver::EmitAvailableEncodedFrames() {
}
const base::TimeTicks now = cast_environment_->Clock()->NowTicks();
const base::TimeTicks playout_time =
GetPlayoutTime(encoded_frame->rtp_timestamp);
const base::TimeTicks playout_time = GetPlayoutTime(*encoded_frame);
// If we have multiple decodable frames, and the current frame is
// too old, then skip it and decode the next frame instead.
......@@ -251,6 +250,10 @@ void FrameReceiver::EmitAvailableEncodedFrames() {
// At this point, we have a decrypted EncodedFrame ready to be emitted.
encoded_frame->reference_time = playout_time;
framer_.ReleaseFrame(encoded_frame->frame_id);
if (encoded_frame->new_playout_delay_ms) {
target_playout_delay_ = base::TimeDelta::FromMilliseconds(
encoded_frame->new_playout_delay_ms);
}
cast_environment_->PostTask(CastEnvironment::MAIN,
FROM_HERE,
base::Bind(frame_request_queue_.front(),
......@@ -266,13 +269,18 @@ void FrameReceiver::EmitAvailableEncodedFramesAfterWaiting() {
EmitAvailableEncodedFrames();
}
base::TimeTicks FrameReceiver::GetPlayoutTime(uint32 rtp_timestamp) const {
base::TimeTicks FrameReceiver::GetPlayoutTime(const EncodedFrame& frame) const {
base::TimeDelta target_playout_delay = target_playout_delay_;
if (frame.new_playout_delay_ms) {
target_playout_delay = base::TimeDelta::FromMilliseconds(
frame.new_playout_delay_ms);
}
return lip_sync_reference_time_ +
lip_sync_drift_.Current() +
RtpDeltaToTimeDelta(
static_cast<int32>(rtp_timestamp - lip_sync_rtp_timestamp_),
static_cast<int32>(frame.rtp_timestamp - lip_sync_rtp_timestamp_),
rtp_timebase_) +
target_playout_delay_;
target_playout_delay;
}
void FrameReceiver::ScheduleNextCastMessage() {
......
......@@ -92,7 +92,7 @@ class FrameReceiver : public RtpPayloadFeedback,
// Computes the playout time for a frame with the given |rtp_timestamp|.
// Because lip-sync info is refreshed regularly, calling this method with the
// same argument may return different results.
base::TimeTicks GetPlayoutTime(uint32 rtp_timestamp) const;
base::TimeTicks GetPlayoutTime(const EncodedFrame& frame) const;
// Schedule timing for the next cast message.
void ScheduleNextCastMessage();
......@@ -130,7 +130,7 @@ class FrameReceiver : public RtpPayloadFeedback,
// transmit/retransmit, receive, decode, and render; given its run-time
// environment (sender/receiver hardware performance, network conditions,
// etc.).
const base::TimeDelta target_playout_delay_;
base::TimeDelta target_playout_delay_;
// Hack: This is used in logic that determines whether to skip frames.
// TODO(miu): Revisit this. Logic needs to also account for expected decode
......
......@@ -23,16 +23,6 @@ const int kMinSchedulingDelayMs = 1;
// well.
const int kAudioFrameRate = 100;
// Helper function to compute the maximum unacked audio frames that is sent.
int GetMaxUnackedFrames(base::TimeDelta target_delay) {
// As long as it doesn't go over |kMaxUnackedFrames|, it is okay to send more
// audio data than the target delay would suggest. Audio packets are tiny and
// receiver has the ability to drop any one of the packets.
// We send up to three times of the target delay of audio frames.
int frames =
1 + 2 * target_delay * kAudioFrameRate / base::TimeDelta::FromSeconds(1);
return std::min(kMaxUnackedFrames, frames);
}
} // namespace
AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment,
......@@ -43,9 +33,9 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment,
transport_sender,
base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval),
audio_config.frequency,
audio_config.ssrc),
target_playout_delay_(audio_config.target_playout_delay),
max_unacked_frames_(GetMaxUnackedFrames(target_playout_delay_)),
audio_config.ssrc,
kAudioFrameRate * 2.0, // We lie to increase max outstanding frames.
audio_config.target_playout_delay),
configured_encoder_bitrate_(audio_config.bitrate),
num_aggressive_rtcp_reports_sent_(0),
last_sent_frame_id_(0),
......@@ -152,6 +142,10 @@ void AudioSender::SendEncodedAudioFrame(
SendRtcpReport(is_last_aggressive_report);
}
if (send_target_playout_delay_) {
encoded_frame->new_playout_delay_ms =
target_playout_delay_.InMilliseconds();
}
transport_sender_->InsertCodedAudioFrame(*encoded_frame);
}
......
......@@ -73,18 +73,6 @@ class AudioSender : public FrameSender,
// Called by the |audio_encoder_| with the next EncodedFrame to send.
void SendEncodedAudioFrame(scoped_ptr<EncodedFrame> audio_frame);
// The total amount of time between a frame's capture/recording on the sender
// and its playback on the receiver (i.e., shown to a user). This is fixed as
// a value large enough to give the system sufficient time to encode,
// transmit/retransmit, receive, decode, and render; given its run-time
// environment (sender/receiver hardware performance, network conditions,
// etc.).
const base::TimeDelta target_playout_delay_;
// Maximum number of outstanding frames before the encoding and sending of
// new frames shall halt.
const int max_unacked_frames_;
// Encodes AudioBuses into EncodedFrames.
scoped_ptr<AudioEncoder> audio_encoder_;
const int configured_encoder_bitrate_;
......
......@@ -14,14 +14,19 @@ FrameSender::FrameSender(scoped_refptr<CastEnvironment> cast_environment,
CastTransportSender* const transport_sender,
base::TimeDelta rtcp_interval,
int frequency,
uint32 ssrc)
uint32 ssrc,
double max_frame_rate,
base::TimeDelta playout_delay)
: cast_environment_(cast_environment),
transport_sender_(transport_sender),
ssrc_(ssrc),
rtp_timestamp_helper_(frequency),
rtt_available_(false),
rtcp_interval_(rtcp_interval),
max_frame_rate_(max_frame_rate),
weak_factory_(this) {
SetTargetPlayoutDelay(playout_delay);
send_target_playout_delay_ = false;
}
FrameSender::~FrameSender() {
......@@ -68,5 +73,16 @@ void FrameSender::OnReceivedRtt(base::TimeDelta rtt,
max_rtt_ = max_rtt;
}
void FrameSender::SetTargetPlayoutDelay(
base::TimeDelta new_target_playout_delay) {
target_playout_delay_ = new_target_playout_delay;
max_unacked_frames_ =
std::min(kMaxUnackedFrames,
1 + static_cast<int>(target_playout_delay_ *
max_frame_rate_ /
base::TimeDelta::FromSeconds(1)));
send_target_playout_delay_ = true;
}
} // namespace cast
} // namespace media
......@@ -26,9 +26,19 @@ class FrameSender {
CastTransportSender* const transport_sender,
base::TimeDelta rtcp_interval,
int frequency,
uint32 ssrc);
uint32 ssrc,
double max_frame_rate,
base::TimeDelta playout_delay);
virtual ~FrameSender();
// Calling this function is only valid if the receiver supports the
// "extra_playout_delay", rtp extension.
void SetTargetPlayoutDelay(base::TimeDelta new_target_playout_delay);
base::TimeDelta GetTargetPlayoutDelay() const {
return target_playout_delay_;
}
protected:
// Schedule and execute periodic sending of RTCP report.
void ScheduleNextRtcpReport();
......@@ -63,9 +73,28 @@ class FrameSender {
base::TimeDelta min_rtt_;
base::TimeDelta max_rtt_;
private:
protected:
const base::TimeDelta rtcp_interval_;
// The total amount of time between a frame's capture/recording on the sender
// and its playback on the receiver (i.e., shown to a user). This is fixed as
// a value large enough to give the system sufficient time to encode,
// transmit/retransmit, receive, decode, and render; given its run-time
// environment (sender/receiver hardware performance, network conditions,
// etc.).
base::TimeDelta target_playout_delay_;
// If true, we transmit the target playout delay to the receiver.
bool send_target_playout_delay_;
// Max encoded frames generated per second.
double max_frame_rate_;
// Maximum number of outstanding frames before the encoding and sending of
// new frames shall halt.
int max_unacked_frames_;
private:
// NOTE: Weak pointers must be invalidated before all other member variables.
base::WeakPtrFactory<FrameSender> weak_factory_;
......
......@@ -47,13 +47,9 @@ VideoSender::VideoSender(
transport_sender,
base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
kVideoFrequency,
video_config.ssrc),
target_playout_delay_(video_config.target_playout_delay),
max_unacked_frames_(
std::min(kMaxUnackedFrames,
1 + static_cast<int>(target_playout_delay_ *
video_config.max_frame_rate /
base::TimeDelta::FromSeconds(1)))),
video_config.ssrc,
video_config.max_frame_rate,
video_config.target_playout_delay),
fixed_bitrate_(GetFixedBitrate(video_config)),
num_aggressive_rtcp_reports_sent_(0),
frames_in_encoder_(0),
......@@ -220,6 +216,10 @@ void VideoSender::SendEncodedVideoFrame(
congestion_control_.SendFrameToTransport(
frame_id, encoded_frame->data.size() * 8, last_send_time_);
if (send_target_playout_delay_) {
encoded_frame->new_playout_delay_ms =
target_playout_delay_.InMilliseconds();
}
transport_sender_->InsertCodedVideoFrame(*encoded_frame);
}
......
......@@ -79,19 +79,6 @@ class VideoSender : public FrameSender,
// Called by the |video_encoder_| with the next EncodeFrame to send.
void SendEncodedVideoFrame(int requested_bitrate_before_encode,
scoped_ptr<EncodedFrame> encoded_frame);
// The total amount of time between a frame's capture/recording on the sender
// and its playback on the receiver (i.e., shown to a user). This is fixed as
// a value large enough to give the system sufficient time to encode,
// transmit/retransmit, receive, decode, and render; given its run-time
// environment (sender/receiver hardware performance, network conditions,
// etc.).
const base::TimeDelta target_playout_delay_;
// Maximum number of outstanding frames before the encoding and sending of
// new frames shall halt.
const int max_unacked_frames_;
// If this value is non zero then a fixed value is used for bitrate.
// If external video encoder is used then bitrate will be fixed to
// (min_bitrate + max_bitrate) / 2.
......
......@@ -1494,6 +1494,38 @@ TEST_F(End2EndTest, OldPacketNetwork) {
EXPECT_EQ(10000ul, video_ticks_.size());
}
TEST_F(End2EndTest, TestSetPlayoutDelay) {
Configure(CODEC_VIDEO_FAKE, CODEC_AUDIO_PCM16, 32000, 1);
Create();
StartBasicPlayer();
const int kNewDelay = 600;
int frames_counter = 0;
for (; frames_counter < 200; ++frames_counter) {
SendFakeVideoFrame(testing_clock_sender_->NowTicks());
RunTasks(kFrameTimerMs);
}
cast_sender_->SetTargetPlayoutDelay(
base::TimeDelta::FromMilliseconds(kNewDelay));
for (; frames_counter < 400; ++frames_counter) {
SendFakeVideoFrame(testing_clock_sender_->NowTicks());
RunTasks(kFrameTimerMs);
}
RunTasks(100 * kFrameTimerMs + 1); // Empty the pipeline.
size_t jump = 0;
for (size_t i = 1; i < video_ticks_.size(); i++) {
int64 delta = (video_ticks_[i].second -
video_ticks_[i-1].second).InMilliseconds();
if (delta > 100) {
EXPECT_EQ(delta, kNewDelay - kTargetPlayoutDelayMs + kFrameTimerMs);
EXPECT_EQ(0u, jump);
jump = i;
}
}
EXPECT_GT(jump, 199u);
EXPECT_LT(jump, 220u);
}
// TODO(pwestin): Add repeatable packet loss test.
// TODO(pwestin): Add test for misaligned send get calls.
// TODO(pwestin): Add more tests that does not resample.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment