Skip to content

GitLab

T
tangled
Commit 47581b1f authored by hubbe's avatar hubbe Committed by Commit bot
Browse files
Options

Cast: Merge common functionality from audio/video sender into frame_sender. 

To make things simpler, also make congestion control into an interface with
fixed and adaptive implementations.

BUG=405622

Review URL: https://codereview.chromium.org/542883004

Cr-Commit-Position: refs/heads/master@{#293630}
parent 41dbfb1a
Hide whitespace changes
Inline Side-by-side
Showing with 414 additions and 454 deletions
media/cast/sender/audio_sender.cc
View file @ 47581b1f
...@@ -15,8 +15,6 @@ namespace media { ...@@ -15,8 +15,6 @@ namespace media {
namespace cast { namespace cast {
namespace { namespace {
const int kNumAggressiveReportsSentAtStart = 100;
// TODO(miu): This should be specified in AudioSenderConfig, but currently it is // TODO(miu): This should be specified in AudioSenderConfig, but currently it is
// fixed to 100 FPS (i.e., 10 ms per frame), and AudioEncoder assumes this as // fixed to 100 FPS (i.e., 10 ms per frame), and AudioEncoder assumes this as
// well. // well.
...@@ -29,13 +27,15 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, ...@@ -29,13 +27,15 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment,
CastTransportSender* const transport_sender) CastTransportSender* const transport_sender)
: FrameSender( : FrameSender(
cast_environment, cast_environment,
true,
transport_sender, transport_sender,
base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval), base::TimeDelta::FromMilliseconds(audio_config.rtcp_interval),
audio_config.frequency, audio_config.frequency,
audio_config.ssrc, audio_config.ssrc,
kAudioFrameRate * 2.0, // We lie to increase max outstanding frames. kAudioFrameRate * 2.0, // We lie to increase max outstanding frames.
audio_config.target_playout_delay), audio_config.target_playout_delay,
configured_encoder_bitrate_(audio_config.bitrate), NewFixedCongestionControl(audio_config.bitrate)),
samples_sent_to_encoder_(0),
weak_factory_(this) { weak_factory_(this) {
cast_initialization_status_ = STATUS_AUDIO_UNINITIALIZED; cast_initialization_status_ = STATUS_AUDIO_UNINITIALIZED;
VLOG(1) << "max_unacked_frames " << max_unacked_frames_; VLOG(1) << "max_unacked_frames " << max_unacked_frames_;
...@@ -48,8 +48,9 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment, ...@@ -48,8 +48,9 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment,
audio_config.frequency, audio_config.frequency,
audio_config.bitrate, audio_config.bitrate,
audio_config.codec, audio_config.codec,
base::Bind(&AudioSender::SendEncodedAudioFrame, base::Bind(&FrameSender::SendEncodedFrame,
weak_factory_.GetWeakPtr()))); weak_factory_.GetWeakPtr(),
audio_config.bitrate)));
cast_initialization_status_ = audio_encoder_->InitializationResult(); cast_initialization_status_ = audio_encoder_->InitializationResult();
} else { } else {
NOTREACHED(); // No support for external audio encoding. NOTREACHED(); // No support for external audio encoding.
...@@ -89,142 +90,17 @@ void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus, ...@@ -89,142 +90,17 @@ void AudioSender::InsertAudio(scoped_ptr<AudioBus> audio_bus,
return; return;
} }
audio_encoder_->InsertAudio(audio_bus.Pass(), recorded_time); int64 old_frames_sent =
} samples_sent_to_encoder_ * kAudioFrameRate / rtp_timebase_;
samples_sent_to_encoder_ += audio_bus->frames();
void AudioSender::SendEncodedAudioFrame( int64 new_frames_sent =
scoped_ptr<EncodedFrame> encoded_frame) { samples_sent_to_encoder_ * kAudioFrameRate / rtp_timebase_;
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN)); frames_in_encoder_ += new_frames_sent - old_frames_sent;
const uint32 frame_id = encoded_frame->frame_id;
const bool is_first_frame_to_be_sent = last_send_time_.is_null();
last_send_time_ = cast_environment_->Clock()->NowTicks();
last_sent_frame_id_ = frame_id;
// If this is the first frame about to be sent, fake the value of
// |latest_acked_frame_id_| to indicate the receiver starts out all caught up.
// Also, schedule the periodic frame re-send checks.
if (is_first_frame_to_be_sent) {
latest_acked_frame_id_ = frame_id - 1;
ScheduleNextResendCheck();
}
cast_environment_->Logging()->InsertEncodedFrameEvent(
last_send_time_, FRAME_ENCODED, AUDIO_EVENT, encoded_frame->rtp_timestamp,
frame_id, static_cast<int>(encoded_frame->data.size()),
encoded_frame->dependency == EncodedFrame::KEY,
configured_encoder_bitrate_);
RecordLatestFrameTimestamps(frame_id,
encoded_frame->reference_time,
encoded_frame->rtp_timestamp);
// At the start of the session, it's important to send reports before each
// frame so that the receiver can properly compute playout times. The reason
// more than one report is sent is because transmission is not guaranteed,
// only best effort, so we send enough that one should almost certainly get
// through.
if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
// SendRtcpReport() will schedule future reports to be made if this is the
// last "aggressive report."
++num_aggressive_rtcp_reports_sent_;
const bool is_last_aggressive_report =
(num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart);
VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report.";
SendRtcpReport(is_last_aggressive_report);
}
if (send_target_playout_delay_) {
encoded_frame->new_playout_delay_ms =
target_playout_delay_.InMilliseconds();
}
transport_sender_->InsertFrame(ssrc_, *encoded_frame);
}
void AudioSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) { audio_encoder_->InsertAudio(audio_bus.Pass(), recorded_time);
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
if (is_rtt_available()) {
// Having the RTT values implies the receiver sent back a receiver report
// based on it having received a report from here. Therefore, ensure this
// sender stops aggressively sending reports.
if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
VLOG(1) << "No longer a need to send reports aggressively (sent "
<< num_aggressive_rtcp_reports_sent_ << ").";
num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart;
ScheduleNextRtcpReport();
}
}
if (last_send_time_.is_null())
return; // Cannot get an ACK without having first sent a frame.
if (cast_feedback.missing_frames_and_packets.empty()) {
// We only count duplicate ACKs when we have sent newer frames.
if (latest_acked_frame_id_ == cast_feedback.ack_frame_id &&
latest_acked_frame_id_ != last_sent_frame_id_) {
duplicate_ack_counter_++;
} else {
duplicate_ack_counter_ = 0;
}
// TODO(miu): The values "2" and "3" should be derived from configuration.
if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) {
VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_;
ResendForKickstart();
}
} else {
// Only count duplicated ACKs if there is no NACK request in between.
// This is to avoid aggresive resend.
duplicate_ack_counter_ = 0;
}
cast_environment_->Logging()->InsertFrameEvent(
cast_environment_->Clock()->NowTicks(),
FRAME_ACK_RECEIVED,
AUDIO_EVENT,
GetRecordedRtpTimestamp(cast_feedback.ack_frame_id),
cast_feedback.ack_frame_id);
const bool is_acked_out_of_order =
static_cast<int32>(cast_feedback.ack_frame_id -
latest_acked_frame_id_) < 0;
VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "")
<< " for frame " << cast_feedback.ack_frame_id;
if (!is_acked_out_of_order) {
// Cancel resends of acked frames.
std::vector<uint32> cancel_sending_frames;
while (latest_acked_frame_id_ != cast_feedback.ack_frame_id) {
latest_acked_frame_id_++;
cancel_sending_frames.push_back(latest_acked_frame_id_);
}
transport_sender_->CancelSendingFrames(ssrc_, cancel_sending_frames);
latest_acked_frame_id_ = cast_feedback.ack_frame_id;
}
} }
bool AudioSender::ShouldDropNextFrame(base::TimeTicks capture_time) const { void AudioSender::OnAck(uint32 frame_id) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
int frames_in_flight = 0;
base::TimeDelta duration_in_flight;
if (!last_send_time_.is_null()) {
frames_in_flight =
static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_);
if (frames_in_flight > 0) {
const uint32 oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
duration_in_flight =
capture_time - GetRecordedReferenceTime(oldest_unacked_frame_id);
}
}
VLOG(2) << frames_in_flight
<< " frames in flight; last sent: " << last_sent_frame_id_
<< "; latest acked: " << latest_acked_frame_id_
<< "; duration in flight: "
<< duration_in_flight.InMicroseconds() << " usec ("
<< (target_playout_delay_ > base::TimeDelta() ?
100 * duration_in_flight / target_playout_delay_ :
kint64max) << "%)";
return frames_in_flight >= max_unacked_frames_ ||
duration_in_flight >= target_playout_delay_;
} }
} // namespace cast } // namespace cast
......
media/cast/sender/audio_sender.h
View file @ 47581b1f
...@@ -51,22 +51,17 @@ class AudioSender : public FrameSender, ...@@ -51,22 +51,17 @@ class AudioSender : public FrameSender,
const base::TimeTicks& recorded_time); const base::TimeTicks& recorded_time);
protected: protected:
// Protected for testability. virtual void OnAck(uint32 frame_id) OVERRIDE;
void OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback);
private: private:
// Returns true if there are too many frames in flight, or if the media
// duration of the frames in flight would be too high by sending the next
// frame. The latter metric is determined from the given |capture_time|
// for the next frame to be encoded and sent.
bool ShouldDropNextFrame(base::TimeTicks capture_time) const;
// Called by the |audio_encoder_| with the next EncodedFrame to send. // Called by the |audio_encoder_| with the next EncodedFrame to send.
void SendEncodedAudioFrame(scoped_ptr<EncodedFrame> audio_frame); void SendEncodedAudioFrame(int requested_bitrate_before_encode,
scoped_ptr<EncodedFrame> audio_frame);
// Encodes AudioBuses into EncodedFrames. // Encodes AudioBuses into EncodedFrames.
scoped_ptr<AudioEncoder> audio_encoder_; scoped_ptr<AudioEncoder> audio_encoder_;
const int configured_encoder_bitrate_;
uint64 samples_sent_to_encoder_;
// NOTE: Weak pointers must be invalidated before all other member variables. // NOTE: Weak pointers must be invalidated before all other member variables.
base::WeakPtrFactory<AudioSender> weak_factory_; base::WeakPtrFactory<AudioSender> weak_factory_;
......
media/cast/sender/congestion_control.cc
View file @ 47581b1f
...@@ -22,6 +22,117 @@ ...@@ -22,6 +22,117 @@
namespace media { namespace media {
namespace cast { namespace cast {
class AdaptiveCongestionControl : public CongestionControl {
public:
AdaptiveCongestionControl(base::TickClock* clock,
uint32 max_bitrate_configured,
uint32 min_bitrate_configured,
size_t max_unacked_frames);
virtual ~AdaptiveCongestionControl() OVERRIDE;
virtual void UpdateRtt(base::TimeDelta rtt) OVERRIDE;
// Called when an encoded frame is sent to the transport.
virtual void SendFrameToTransport(uint32 frame_id,
size_t frame_size,
base::TimeTicks when) OVERRIDE;
// Called when we receive an ACK for a frame.
virtual void AckFrame(uint32 frame_id, base::TimeTicks when) OVERRIDE;
// Returns the bitrate we should use for the next frame.
virtual uint32 GetBitrate(base::TimeTicks playout_time,
base::TimeDelta playout_delay) OVERRIDE;
private:
struct FrameStats {
FrameStats();
// Time this frame was sent to the transport.
base::TimeTicks sent_time;
// Time this frame was acked.
base::TimeTicks ack_time;
// Size of encoded frame in bits.
size_t frame_size;
};
// Calculate how much "dead air" (idle time) there is between two frames.
static base::TimeDelta DeadTime(const FrameStats& a, const FrameStats& b);
// Get the FrameStats for a given |frame_id|.
// Note: Older FrameStats will be removed automatically.
FrameStats* GetFrameStats(uint32 frame_id);
// Calculate a safe bitrate. This is based on how much we've been
// sending in the past.
double CalculateSafeBitrate();
// For a given frame, calculate when it might be acked.
// (Or return the time it was acked, if it was.)
base::TimeTicks EstimatedAckTime(uint32 frame_id, double bitrate);
// Calculate when we start sending the data for a given frame.
// This is done by calculating when we were done sending the previous
// frame, but obviously can't be less than |sent_time| (if known).
base::TimeTicks EstimatedSendingTime(uint32 frame_id, double bitrate);
base::TickClock* const clock_; // Not owned by this class.
const uint32 max_bitrate_configured_;
const uint32 min_bitrate_configured_;
std::deque<FrameStats> frame_stats_;
uint32 last_frame_stats_;
uint32 last_acked_frame_;
uint32 last_encoded_frame_;
base::TimeDelta rtt_;
size_t history_size_;
size_t acked_bits_in_history_;
base::TimeDelta dead_time_in_history_;
DISALLOW_COPY_AND_ASSIGN(AdaptiveCongestionControl);
};
class FixedCongestionControl : public CongestionControl {
public:
FixedCongestionControl(uint32 bitrate) : bitrate_(bitrate) {}
virtual ~FixedCongestionControl() OVERRIDE {}
virtual void UpdateRtt(base::TimeDelta rtt) OVERRIDE {
}
// Called when an encoded frame is sent to the transport.
virtual void SendFrameToTransport(uint32 frame_id,
size_t frame_size,
base::TimeTicks when) OVERRIDE {
}
// Called when we receive an ACK for a frame.
virtual void AckFrame(uint32 frame_id, base::TimeTicks when) OVERRIDE {
}
// Returns the bitrate we should use for the next frame.
virtual uint32 GetBitrate(base::TimeTicks playout_time,
base::TimeDelta playout_delay) OVERRIDE {
return bitrate_;
}
private:
uint32 bitrate_;
DISALLOW_COPY_AND_ASSIGN(FixedCongestionControl);
};
CongestionControl* NewAdaptiveCongestionControl(
base::TickClock* clock,
uint32 max_bitrate_configured,
uint32 min_bitrate_configured,
size_t max_unacked_frames) {
return new AdaptiveCongestionControl(clock,
max_bitrate_configured,
min_bitrate_configured,
max_unacked_frames);
}
CongestionControl* NewFixedCongestionControl(uint32 bitrate) {
return new FixedCongestionControl(bitrate);
}
// This means that we *try* to keep our buffer 90% empty. // This means that we *try* to keep our buffer 90% empty.
// If it is less full, we increase the bandwidth, if it is more // If it is less full, we increase the bandwidth, if it is more
// we decrease the bandwidth. Making this smaller makes the // we decrease the bandwidth. Making this smaller makes the
...@@ -32,13 +143,14 @@ static const double kTargetEmptyBufferFraction = 0.9; ...@@ -32,13 +143,14 @@ static const double kTargetEmptyBufferFraction = 0.9;
// congestion control adapt slower. // congestion control adapt slower.
static const size_t kHistorySize = 100; static const size_t kHistorySize = 100;
CongestionControl::FrameStats::FrameStats() : frame_size(0) { AdaptiveCongestionControl::FrameStats::FrameStats() : frame_size(0) {
} }
CongestionControl::CongestionControl(base::TickClock* clock, AdaptiveCongestionControl::AdaptiveCongestionControl(
uint32 max_bitrate_configured, base::TickClock* clock,
uint32 min_bitrate_configured, uint32 max_bitrate_configured,
size_t max_unacked_frames) uint32 min_bitrate_configured,
size_t max_unacked_frames)
: clock_(clock), : clock_(clock),
max_bitrate_configured_(max_bitrate_configured), max_bitrate_configured_(max_bitrate_configured),
min_bitrate_configured_(min_bitrate_configured), min_bitrate_configured_(min_bitrate_configured),
...@@ -57,14 +169,15 @@ CongestionControl::CongestionControl(base::TickClock* clock, ...@@ -57,14 +169,15 @@ CongestionControl::CongestionControl(base::TickClock* clock,
} }
CongestionControl::~CongestionControl() {} CongestionControl::~CongestionControl() {}
AdaptiveCongestionControl::~AdaptiveCongestionControl() {}
void CongestionControl::UpdateRtt(base::TimeDelta rtt) { void AdaptiveCongestionControl::UpdateRtt(base::TimeDelta rtt) {
rtt_ = (7 * rtt_ + rtt) / 8; rtt_ = (7 * rtt_ + rtt) / 8;
} }
// Calculate how much "dead air" there is between two frames. // Calculate how much "dead air" there is between two frames.
base::TimeDelta CongestionControl::DeadTime(const FrameStats& a, base::TimeDelta AdaptiveCongestionControl::DeadTime(const FrameStats& a,
const FrameStats& b) { const FrameStats& b) {
if (b.sent_time > a.ack_time) { if (b.sent_time > a.ack_time) {
return b.sent_time - a.ack_time; return b.sent_time - a.ack_time;
} else { } else {
...@@ -72,7 +185,7 @@ base::TimeDelta CongestionControl::DeadTime(const FrameStats& a, ...@@ -72,7 +185,7 @@ base::TimeDelta CongestionControl::DeadTime(const FrameStats& a,
} }
} }
double CongestionControl::CalculateSafeBitrate() { double AdaptiveCongestionControl::CalculateSafeBitrate() {
double transmit_time = double transmit_time =
(GetFrameStats(last_acked_frame_)->ack_time - (GetFrameStats(last_acked_frame_)->ack_time -
frame_stats_.front().sent_time - dead_time_in_history_).InSecondsF(); frame_stats_.front().sent_time - dead_time_in_history_).InSecondsF();
...@@ -83,8 +196,8 @@ double CongestionControl::CalculateSafeBitrate() { ...@@ -83,8 +196,8 @@ double CongestionControl::CalculateSafeBitrate() {
return acked_bits_in_history_ / std::max(transmit_time, 1E-3); return acked_bits_in_history_ / std::max(transmit_time, 1E-3);
} }
CongestionControl::FrameStats* CongestionControl::GetFrameStats( AdaptiveCongestionControl::FrameStats*
uint32 frame_id) { AdaptiveCongestionControl::GetFrameStats(uint32 frame_id) {
int32 offset = static_cast<int32>(frame_id - last_frame_stats_); int32 offset = static_cast<int32>(frame_id - last_frame_stats_);
DCHECK_LT(offset, static_cast<int32>(kHistorySize)); DCHECK_LT(offset, static_cast<int32>(kHistorySize));
if (offset > 0) { if (offset > 0) {
...@@ -109,7 +222,8 @@ CongestionControl::FrameStats* CongestionControl::GetFrameStats( ...@@ -109,7 +222,8 @@ CongestionControl::FrameStats* CongestionControl::GetFrameStats(
return &frame_stats_[offset]; return &frame_stats_[offset];
} }
void CongestionControl::AckFrame(uint32 frame_id, base::TimeTicks when) { void AdaptiveCongestionControl::AckFrame(uint32 frame_id,
base::TimeTicks when) {
FrameStats* frame_stats = GetFrameStats(last_acked_frame_); FrameStats* frame_stats = GetFrameStats(last_acked_frame_);
while (IsNewerFrameId(frame_id, last_acked_frame_)) { while (IsNewerFrameId(frame_id, last_acked_frame_)) {
FrameStats* last_frame_stats = frame_stats; FrameStats* last_frame_stats = frame_stats;
...@@ -129,9 +243,9 @@ void CongestionControl::AckFrame(uint32 frame_id, base::TimeTicks when) { ...@@ -129,9 +243,9 @@ void CongestionControl::AckFrame(uint32 frame_id, base::TimeTicks when) {
} }
} }
void CongestionControl::SendFrameToTransport(uint32 frame_id, void AdaptiveCongestionControl::SendFrameToTransport(uint32 frame_id,
size_t frame_size, size_t frame_size,
base::TimeTicks when) { base::TimeTicks when) {
last_encoded_frame_ = frame_id; last_encoded_frame_ = frame_id;
FrameStats* frame_stats = GetFrameStats(frame_id); FrameStats* frame_stats = GetFrameStats(frame_id);
DCHECK(frame_stats); DCHECK(frame_stats);
...@@ -139,8 +253,8 @@ void CongestionControl::SendFrameToTransport(uint32 frame_id, ...@@ -139,8 +253,8 @@ void CongestionControl::SendFrameToTransport(uint32 frame_id,
frame_stats->sent_time = when; frame_stats->sent_time = when;
} }
base::TimeTicks CongestionControl::EstimatedAckTime(uint32 frame_id, base::TimeTicks AdaptiveCongestionControl::EstimatedAckTime(uint32 frame_id,
double bitrate) { double bitrate) {
FrameStats* frame_stats = GetFrameStats(frame_id); FrameStats* frame_stats = GetFrameStats(frame_id);
DCHECK(frame_stats); DCHECK(frame_stats);
if (frame_stats->ack_time.is_null()) { if (frame_stats->ack_time.is_null()) {
...@@ -164,8 +278,9 @@ base::TimeTicks CongestionControl::EstimatedAckTime(uint32 frame_id, ...@@ -164,8 +278,9 @@ base::TimeTicks CongestionControl::EstimatedAckTime(uint32 frame_id,
} }
} }
base::TimeTicks CongestionControl::EstimatedSendingTime(uint32 frame_id, base::TimeTicks AdaptiveCongestionControl::EstimatedSendingTime(
double bitrate) { uint32 frame_id,
double bitrate) {
FrameStats* frame_stats = GetFrameStats(frame_id); FrameStats* frame_stats = GetFrameStats(frame_id);
DCHECK(frame_stats); DCHECK(frame_stats);
base::TimeTicks ret = EstimatedAckTime(frame_id - 1, bitrate) - rtt_; base::TimeTicks ret = EstimatedAckTime(frame_id - 1, bitrate) - rtt_;
...@@ -177,8 +292,8 @@ base::TimeTicks CongestionControl::EstimatedSendingTime(uint32 frame_id, ...@@ -177,8 +292,8 @@ base::TimeTicks CongestionControl::EstimatedSendingTime(uint32 frame_id,
} }
} }
uint32 CongestionControl::GetBitrate(base::TimeTicks playout_time, uint32 AdaptiveCongestionControl::GetBitrate(base::TimeTicks playout_time,
base::TimeDelta playout_delay) { base::TimeDelta playout_delay) {
double safe_bitrate = CalculateSafeBitrate(); double safe_bitrate = CalculateSafeBitrate();
// Estimate when we might start sending the next frame. // Estimate when we might start sending the next frame.
base::TimeDelta time_to_catch_up = base::TimeDelta time_to_catch_up =
......
media/cast/sender/congestion_control.h
View file @ 47581b1f
...@@ -17,69 +17,30 @@ namespace cast { ...@@ -17,69 +17,30 @@ namespace cast {
class CongestionControl { class CongestionControl {
public: public:
CongestionControl(base::TickClock* clock,
uint32 max_bitrate_configured,
uint32 min_bitrate_configured,
size_t max_unacked_frames);
virtual ~CongestionControl(); virtual ~CongestionControl();
void UpdateRtt(base::TimeDelta rtt); // Called with latest measured rtt value.
virtual void UpdateRtt(base::TimeDelta rtt) = 0;
// Called when an encoded frame is sent to the transport. // Called when an encoded frame is sent to the transport.
void SendFrameToTransport(uint32 frame_id, virtual void SendFrameToTransport(uint32 frame_id,
size_t frame_size, size_t frame_size,
base::TimeTicks when); base::TimeTicks when) = 0;
// Called when we receive an ACK for a frame. // Called when we receive an ACK for a frame.
void AckFrame(uint32 frame_id, base::TimeTicks when); virtual void AckFrame(uint32 frame_id, base::TimeTicks when) = 0;
// Returns the bitrate we should use for the next frame. // Returns the bitrate we should use for the next frame.
uint32 GetBitrate(base::TimeTicks playout_time, virtual uint32 GetBitrate(base::TimeTicks playout_time,
base::TimeDelta playout_delay); base::TimeDelta playout_delay) = 0;
};
private:
struct FrameStats {
FrameStats();
// Time this frame was sent to the transport.
base::TimeTicks sent_time;
// Time this frame was acked.
base::TimeTicks ack_time;
// Size of encoded frame in bits.
size_t frame_size;
};
// Calculate how much "dead air" (idle time) there is between two frames.
static base::TimeDelta DeadTime(const FrameStats& a, const FrameStats& b);
// Get the FrameStats for a given |frame_id|.
// Note: Older FrameStats will be removed automatically.
FrameStats* GetFrameStats(uint32 frame_id);
// Calculate a safe bitrate. This is based on how much we've been
// sending in the past.
double CalculateSafeBitrate();
// For a given frame, calculate when it might be acked.
// (Or return the time it was acked, if it was.)
base::TimeTicks EstimatedAckTime(uint32 frame_id, double bitrate);
// Calculate when we start sending the data for a given frame.
// This is done by calculating when we were done sending the previous
// frame, but obviously can't be less than |sent_time| (if known).
base::TimeTicks EstimatedSendingTime(uint32 frame_id, double bitrate);
base::TickClock* const clock_; // Not owned by this class. CongestionControl* NewAdaptiveCongestionControl(
const uint32 max_bitrate_configured_; base::TickClock* clock,
const uint32 min_bitrate_configured_; uint32 max_bitrate_configured,
std::deque<FrameStats> frame_stats_; uint32 min_bitrate_configured,
uint32 last_frame_stats_; size_t max_unacked_frames);
uint32 last_acked_frame_;
uint32 last_encoded_frame_;
base::TimeDelta rtt_;
size_t history_size_;
size_t acked_bits_in_history_;
base::TimeDelta dead_time_in_history_;
DISALLOW_COPY_AND_ASSIGN(CongestionControl); CongestionControl* NewFixedCongestionControl(uint32 bitrate);
};
} // namespace cast } // namespace cast
} // namespace media } // namespace media
......
media/cast/sender/congestion_control_unittest.cc
View file @ 47581b1f
...@@ -25,7 +25,7 @@ class CongestionControlTest : public ::testing::Test { ...@@ -25,7 +25,7 @@ class CongestionControlTest : public ::testing::Test {
: task_runner_(new test::FakeSingleThreadTaskRunner(&testing_clock_)) { : task_runner_(new test::FakeSingleThreadTaskRunner(&testing_clock_)) {
testing_clock_.Advance( testing_clock_.Advance(
base::TimeDelta::FromMilliseconds(kStartMillisecond)); base::TimeDelta::FromMilliseconds(kStartMillisecond));
congestion_control_.reset(new CongestionControl( congestion_control_.reset(NewAdaptiveCongestionControl(
&testing_clock_, kMaxBitrateConfigured, kMinBitrateConfigured, 10)); &testing_clock_, kMaxBitrateConfigured, kMinBitrateConfigured, 10));
} }
......
media/cast/sender/frame_sender.cc
View file @ 47581b1f
...@@ -4,30 +4,40 @@ ...@@ -4,30 +4,40 @@
#include "media/cast/sender/frame_sender.h" #include "media/cast/sender/frame_sender.h"
#include "base/debug/trace_event.h"
namespace media { namespace media {
namespace cast { namespace cast {
namespace { namespace {
const int kMinSchedulingDelayMs = 1; const int kMinSchedulingDelayMs = 1;
const int kNumAggressiveReportsSentAtStart = 100;
} // namespace } // namespace
FrameSender::FrameSender(scoped_refptr<CastEnvironment> cast_environment, FrameSender::FrameSender(scoped_refptr<CastEnvironment> cast_environment,
bool is_audio,
CastTransportSender* const transport_sender, CastTransportSender* const transport_sender,
base::TimeDelta rtcp_interval, base::TimeDelta rtcp_interval,
int rtp_timebase, int rtp_timebase,
uint32 ssrc, uint32 ssrc,
double max_frame_rate, double max_frame_rate,
base::TimeDelta playout_delay) base::TimeDelta playout_delay,
CongestionControl* congestion_control)
: cast_environment_(cast_environment), : cast_environment_(cast_environment),
transport_sender_(transport_sender), transport_sender_(transport_sender),
ssrc_(ssrc), ssrc_(ssrc),
rtt_available_(false), rtt_available_(false),
rtcp_interval_(rtcp_interval), rtcp_interval_(rtcp_interval),
max_frame_rate_(max_frame_rate), max_frame_rate_(max_frame_rate),
frames_in_encoder_(0),
num_aggressive_rtcp_reports_sent_(0), num_aggressive_rtcp_reports_sent_(0),
last_sent_frame_id_(0), last_sent_frame_id_(0),
latest_acked_frame_id_(0), latest_acked_frame_id_(0),
duplicate_ack_counter_(0), duplicate_ack_counter_(0),
rtp_timebase_(rtp_timebase), rtp_timebase_(rtp_timebase),
congestion_control_(congestion_control),
is_audio_(is_audio),
weak_factory_(this) { weak_factory_(this) {
DCHECK_GT(rtp_timebase_, 0); DCHECK_GT(rtp_timebase_, 0);
SetTargetPlayoutDelay(playout_delay); SetTargetPlayoutDelay(playout_delay);
...@@ -158,5 +168,185 @@ RtpTimestamp FrameSender::GetRecordedRtpTimestamp(uint32 frame_id) const { ...@@ -158,5 +168,185 @@ RtpTimestamp FrameSender::GetRecordedRtpTimestamp(uint32 frame_id) const {
return frame_rtp_timestamps_[frame_id % arraysize(frame_rtp_timestamps_)]; return frame_rtp_timestamps_[frame_id % arraysize(frame_rtp_timestamps_)];
} }
void FrameSender::SendEncodedFrame(
int requested_bitrate_before_encode,
scoped_ptr<EncodedFrame> encoded_frame) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
DCHECK_GT(frames_in_encoder_, 0) << " is_audio: " << is_audio_;
frames_in_encoder_--;
const uint32 frame_id = encoded_frame->frame_id;
const bool is_first_frame_to_be_sent = last_send_time_.is_null();
last_send_time_ = cast_environment_->Clock()->NowTicks();
last_sent_frame_id_ = frame_id;
// If this is the first frame about to be sent, fake the value of
// |latest_acked_frame_id_| to indicate the receiver starts out all caught up.
// Also, schedule the periodic frame re-send checks.
if (is_first_frame_to_be_sent) {
latest_acked_frame_id_ = frame_id - 1;
ScheduleNextResendCheck();
}
VLOG_IF(1, encoded_frame->dependency == EncodedFrame::KEY)
<< "Send encoded key frame; frame_id: " << frame_id;
cast_environment_->Logging()->InsertEncodedFrameEvent(
last_send_time_, FRAME_ENCODED,
is_audio_ ? AUDIO_EVENT : VIDEO_EVENT,
encoded_frame->rtp_timestamp,
frame_id, static_cast<int>(encoded_frame->data.size()),
encoded_frame->dependency == EncodedFrame::KEY,
requested_bitrate_before_encode);
RecordLatestFrameTimestamps(frame_id,
encoded_frame->reference_time,
encoded_frame->rtp_timestamp);
if (!is_audio_) {
// Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
TRACE_EVENT_INSTANT1(
"cast_perf_test", "VideoFrameEncoded",
TRACE_EVENT_SCOPE_THREAD,
"rtp_timestamp", encoded_frame->rtp_timestamp);
}
// At the start of the session, it's important to send reports before each
// frame so that the receiver can properly compute playout times. The reason
// more than one report is sent is because transmission is not guaranteed,
// only best effort, so send enough that one should almost certainly get
// through.
if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
// SendRtcpReport() will schedule future reports to be made if this is the
// last "aggressive report."
++num_aggressive_rtcp_reports_sent_;
const bool is_last_aggressive_report =
(num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart);
VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report.";
SendRtcpReport(is_last_aggressive_report);
}
congestion_control_->SendFrameToTransport(
frame_id, encoded_frame->data.size() * 8, last_send_time_);
if (send_target_playout_delay_) {
encoded_frame->new_playout_delay_ms =
target_playout_delay_.InMilliseconds();
}
transport_sender_->InsertFrame(ssrc_, *encoded_frame);
}
void FrameSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
base::TimeDelta rtt;
base::TimeDelta avg_rtt;
base::TimeDelta min_rtt;
base::TimeDelta max_rtt;
if (is_rtt_available()) {
rtt = rtt_;
avg_rtt = avg_rtt_;
min_rtt = min_rtt_;
max_rtt = max_rtt_;
congestion_control_->UpdateRtt(rtt);
// Don't use a RTT lower than our average.
rtt = std::max(rtt, avg_rtt);
// Having the RTT values implies the receiver sent back a receiver report
// based on it having received a report from here. Therefore, ensure this
// sender stops aggressively sending reports.
if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
VLOG(1) << "No longer a need to send reports aggressively (sent "
<< num_aggressive_rtcp_reports_sent_ << ").";
num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart;
ScheduleNextRtcpReport();
}
} else {
// We have no measured value use default.
rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
}
if (last_send_time_.is_null())
return; // Cannot get an ACK without having first sent a frame.
if (cast_feedback.missing_frames_and_packets.empty()) {
OnAck(cast_feedback.ack_frame_id);
// We only count duplicate ACKs when we have sent newer frames.
if (latest_acked_frame_id_ == cast_feedback.ack_frame_id &&
latest_acked_frame_id_ != last_sent_frame_id_) {
duplicate_ack_counter_++;
} else {
duplicate_ack_counter_ = 0;
}
// TODO(miu): The values "2" and "3" should be derived from configuration.
if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) {
VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_;
ResendForKickstart();
}
} else {
// Only count duplicated ACKs if there is no NACK request in between.
// This is to avoid aggresive resend.
duplicate_ack_counter_ = 0;
}
base::TimeTicks now = cast_environment_->Clock()->NowTicks();
congestion_control_->AckFrame(cast_feedback.ack_frame_id, now);
cast_environment_->Logging()->InsertFrameEvent(
now,
FRAME_ACK_RECEIVED,
is_audio_ ? AUDIO_EVENT : VIDEO_EVENT,
GetRecordedRtpTimestamp(cast_feedback.ack_frame_id),
cast_feedback.ack_frame_id);
const bool is_acked_out_of_order =
static_cast<int32>(cast_feedback.ack_frame_id -
latest_acked_frame_id_) < 0;
VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "")
<< " for frame " << cast_feedback.ack_frame_id;
if (!is_acked_out_of_order) {
// Cancel resends of acked frames.
std::vector<uint32> cancel_sending_frames;
while (latest_acked_frame_id_ != cast_feedback.ack_frame_id) {
latest_acked_frame_id_++;
cancel_sending_frames.push_back(latest_acked_frame_id_);
}
transport_sender_->CancelSendingFrames(ssrc_, cancel_sending_frames);
latest_acked_frame_id_ = cast_feedback.ack_frame_id;
}
}
bool FrameSender::ShouldDropNextFrame(base::TimeTicks capture_time) const {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
int frames_in_flight = 0;
base::TimeDelta duration_in_flight;
if (!last_send_time_.is_null()) {
frames_in_flight =
static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_);
if (frames_in_flight > 0) {
const uint32 oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
duration_in_flight =
capture_time - GetRecordedReferenceTime(oldest_unacked_frame_id);
}
}
frames_in_flight += frames_in_encoder_;
VLOG(2) << frames_in_flight
<< " frames in flight; last sent: " << last_sent_frame_id_
<< "; latest acked: " << latest_acked_frame_id_
<< "; frames in encoder: " << frames_in_encoder_
<< "; duration in flight: "
<< duration_in_flight.InMicroseconds() << " usec ("
<< (target_playout_delay_ > base::TimeDelta() ?
100 * duration_in_flight / target_playout_delay_ :
kint64max) << "%)";
return frames_in_flight >= max_unacked_frames_ ||
duration_in_flight >= target_playout_delay_;
}
} // namespace cast } // namespace cast
} // namespace media } // namespace media
media/cast/sender/frame_sender.h
View file @ 47581b1f
...@@ -15,6 +15,7 @@ ...@@ -15,6 +15,7 @@
#include "base/time/time.h" #include "base/time/time.h"
#include "media/cast/cast_environment.h" #include "media/cast/cast_environment.h"
#include "media/cast/net/rtcp/rtcp.h" #include "media/cast/net/rtcp/rtcp.h"
#include "media/cast/sender/congestion_control.h"
namespace media { namespace media {
namespace cast { namespace cast {
...@@ -22,12 +23,14 @@ namespace cast { ...@@ -22,12 +23,14 @@ namespace cast {
class FrameSender { class FrameSender {
public: public:
FrameSender(scoped_refptr<CastEnvironment> cast_environment, FrameSender(scoped_refptr<CastEnvironment> cast_environment,
bool is_audio,
CastTransportSender* const transport_sender, CastTransportSender* const transport_sender,
base::TimeDelta rtcp_interval, base::TimeDelta rtcp_interval,
int rtp_timebase, int rtp_timebase,
uint32 ssrc, uint32 ssrc,
double max_frame_rate, double max_frame_rate,
base::TimeDelta playout_delay); base::TimeDelta playout_delay,
CongestionControl* congestion_control);
virtual ~FrameSender(); virtual ~FrameSender();
// Calling this function is only valid if the receiver supports the // Calling this function is only valid if the receiver supports the
...@@ -38,6 +41,10 @@ class FrameSender { ...@@ -38,6 +41,10 @@ class FrameSender {
return target_playout_delay_; return target_playout_delay_;
} }
// Called by the encoder with the next EncodeFrame to send.
void SendEncodedFrame(int requested_bitrate_before_encode,
scoped_ptr<EncodedFrame> encoded_frame);
protected: protected:
// Schedule and execute periodic sending of RTCP report. // Schedule and execute periodic sending of RTCP report.
void ScheduleNextRtcpReport(); void ScheduleNextRtcpReport();
...@@ -78,6 +85,15 @@ class FrameSender { ...@@ -78,6 +85,15 @@ class FrameSender {
void ResendCheck(); void ResendCheck();
void ResendForKickstart(); void ResendForKickstart();
// Protected for testability.
void OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback);
// Returns true if there are too many frames in flight, or if the media
// duration of the frames in flight would be too high by sending the next
// frame. The latter metric is determined from the given |capture_time|
// for the next frame to be encoded and sent.
bool ShouldDropNextFrame(base::TimeTicks capture_time) const;
// Record or retrieve a recent history of each frame's timestamps. // Record or retrieve a recent history of each frame's timestamps.
// Warning: If a frame ID too far in the past is requested, the getters will // Warning: If a frame ID too far in the past is requested, the getters will
// silently succeed but return incorrect values. Be sure to respect // silently succeed but return incorrect values. Be sure to respect
...@@ -88,6 +104,9 @@ class FrameSender { ...@@ -88,6 +104,9 @@ class FrameSender {
base::TimeTicks GetRecordedReferenceTime(uint32 frame_id) const; base::TimeTicks GetRecordedReferenceTime(uint32 frame_id) const;
RtpTimestamp GetRecordedRtpTimestamp(uint32 frame_id) const; RtpTimestamp GetRecordedRtpTimestamp(uint32 frame_id) const;
// Called when we get an ACK for a frame.
virtual void OnAck(uint32 frame_id) = 0;
const base::TimeDelta rtcp_interval_; const base::TimeDelta rtcp_interval_;
// The total amount of time between a frame's capture/recording on the sender // The total amount of time between a frame's capture/recording on the sender
...@@ -108,6 +127,9 @@ class FrameSender { ...@@ -108,6 +127,9 @@ class FrameSender {
// new frames shall halt. // new frames shall halt.
int max_unacked_frames_; int max_unacked_frames_;
// The number of frames currently being processed in |video_encoder_|.
int frames_in_encoder_;
// Counts how many RTCP reports are being "aggressively" sent (i.e., one per // Counts how many RTCP reports are being "aggressively" sent (i.e., one per
// frame) at the start of the session. Once a threshold is reached, RTCP // frame) at the start of the session. Once a threshold is reached, RTCP
// reports are instead sent at the configured interval + random drift. // reports are instead sent at the configured interval + random drift.
...@@ -137,10 +159,16 @@ class FrameSender { ...@@ -137,10 +159,16 @@ class FrameSender {
// STATUS_VIDEO_INITIALIZED. // STATUS_VIDEO_INITIALIZED.
CastInitializationStatus cast_initialization_status_; CastInitializationStatus cast_initialization_status_;
private:
// RTP timestamp increment representing one second. // RTP timestamp increment representing one second.
const int rtp_timebase_; const int rtp_timebase_;
// This object controls how we change the bitrate to make sure the
// buffer doesn't overflow.
scoped_ptr<CongestionControl> congestion_control_;
private:
const bool is_audio_;
// Ring buffers to keep track of recent frame timestamps (both in terms of // Ring buffers to keep track of recent frame timestamps (both in terms of
// local reference time and RTP media time). These should only be accessed // local reference time and RTP media time). These should only be accessed
// through the Record/GetXXX() methods. // through the Record/GetXXX() methods.
......
media/cast/sender/video_sender.cc
View file @ 47581b1f
...@@ -19,22 +19,10 @@ ...@@ -19,22 +19,10 @@
namespace media { namespace media {
namespace cast { namespace cast {
const int kNumAggressiveReportsSentAtStart = 100; // Note, we use a fixed bitrate value when external video encoder is used.
namespace {
// Returns a fixed bitrate value when external video encoder is used.
// Some hardware encoder shows bad behavior if we set the bitrate too // Some hardware encoder shows bad behavior if we set the bitrate too
// frequently, e.g. quality drop, not abiding by target bitrate, etc. // frequently, e.g. quality drop, not abiding by target bitrate, etc.
// See details: crbug.com/392086. // See details: crbug.com/392086.
size_t GetFixedBitrate(const VideoSenderConfig& video_config) {
if (!video_config.use_external_encoder)
return 0;
return (video_config.min_bitrate + video_config.max_bitrate) / 2;
}
} // namespace
VideoSender::VideoSender( VideoSender::VideoSender(
scoped_refptr<CastEnvironment> cast_environment, scoped_refptr<CastEnvironment> cast_environment,
const VideoSenderConfig& video_config, const VideoSenderConfig& video_config,
...@@ -43,18 +31,16 @@ VideoSender::VideoSender( ...@@ -43,18 +31,16 @@ VideoSender::VideoSender(
CastTransportSender* const transport_sender) CastTransportSender* const transport_sender)
: FrameSender( : FrameSender(
cast_environment, cast_environment,
false,
transport_sender, transport_sender,
base::TimeDelta::FromMilliseconds(video_config.rtcp_interval), base::TimeDelta::FromMilliseconds(video_config.rtcp_interval),
kVideoFrequency, kVideoFrequency,
video_config.ssrc, video_config.ssrc,
video_config.max_frame_rate, video_config.max_frame_rate,
video_config.target_playout_delay), video_config.target_playout_delay,
fixed_bitrate_(GetFixedBitrate(video_config)), NewFixedCongestionControl(
frames_in_encoder_(0), (video_config.min_bitrate + video_config.max_bitrate) / 2)),
congestion_control_(cast_environment->Clock(), last_bitrate_(0),
video_config.max_bitrate,
video_config.min_bitrate,
max_unacked_frames_),
weak_factory_(this) { weak_factory_(this) {
cast_initialization_status_ = STATUS_VIDEO_UNINITIALIZED; cast_initialization_status_ = STATUS_VIDEO_UNINITIALIZED;
VLOG(1) << "max_unacked_frames is " << max_unacked_frames_ VLOG(1) << "max_unacked_frames is " << max_unacked_frames_
...@@ -69,6 +55,11 @@ VideoSender::VideoSender( ...@@ -69,6 +55,11 @@ VideoSender::VideoSender(
create_vea_cb, create_vea_cb,
create_video_encode_mem_cb)); create_video_encode_mem_cb));
} else { } else {
congestion_control_.reset(
NewAdaptiveCongestionControl(cast_environment->Clock(),
video_config.max_bitrate,
video_config.min_bitrate,
max_unacked_frames_));
video_encoder_.reset(new VideoEncoderImpl( video_encoder_.reset(new VideoEncoderImpl(
cast_environment, video_config, max_unacked_frames_)); cast_environment, video_config, max_unacked_frames_));
} }
...@@ -124,23 +115,17 @@ void VideoSender::InsertRawVideoFrame( ...@@ -124,23 +115,17 @@ void VideoSender::InsertRawVideoFrame(
return; return;
} }
uint32 bitrate = fixed_bitrate_; uint32 bitrate = congestion_control_->GetBitrate(
if (!bitrate) {
bitrate = congestion_control_.GetBitrate(
capture_time + target_playout_delay_, target_playout_delay_); capture_time + target_playout_delay_, target_playout_delay_);
DCHECK(bitrate); if (bitrate != last_bitrate_) {
video_encoder_->SetBitRate(bitrate);
} else if (last_send_time_.is_null()) {
// Set the fixed bitrate value to codec until a frame is sent. We might
// set this value a couple times at the very beginning of the stream but
// it is not harmful.
video_encoder_->SetBitRate(bitrate); video_encoder_->SetBitRate(bitrate);
last_bitrate_ = bitrate;
} }
if (video_encoder_->EncodeVideoFrame( if (video_encoder_->EncodeVideoFrame(
video_frame, video_frame,
capture_time, capture_time,
base::Bind(&VideoSender::SendEncodedVideoFrame, base::Bind(&FrameSender::SendEncodedFrame,
weak_factory_.GetWeakPtr(), weak_factory_.GetWeakPtr(),
bitrate))) { bitrate))) {
frames_in_encoder_++; frames_in_encoder_++;
...@@ -149,179 +134,8 @@ void VideoSender::InsertRawVideoFrame( ...@@ -149,179 +134,8 @@ void VideoSender::InsertRawVideoFrame(
} }
} }
void VideoSender::SendEncodedVideoFrame( void VideoSender::OnAck(uint32 frame_id) {
int requested_bitrate_before_encode, video_encoder_->LatestFrameIdToReference(frame_id);
scoped_ptr<EncodedFrame> encoded_frame) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
DCHECK_GT(frames_in_encoder_, 0);
frames_in_encoder_--;
const uint32 frame_id = encoded_frame->frame_id;
const bool is_first_frame_to_be_sent = last_send_time_.is_null();
last_send_time_ = cast_environment_->Clock()->NowTicks();
last_sent_frame_id_ = frame_id;
// If this is the first frame about to be sent, fake the value of
// |latest_acked_frame_id_| to indicate the receiver starts out all caught up.
// Also, schedule the periodic frame re-send checks.
if (is_first_frame_to_be_sent) {
latest_acked_frame_id_ = frame_id - 1;
ScheduleNextResendCheck();
}
VLOG_IF(1, encoded_frame->dependency == EncodedFrame::KEY)
<< "Send encoded key frame; frame_id: " << frame_id;
cast_environment_->Logging()->InsertEncodedFrameEvent(
last_send_time_, FRAME_ENCODED, VIDEO_EVENT, encoded_frame->rtp_timestamp,
frame_id, static_cast<int>(encoded_frame->data.size()),
encoded_frame->dependency == EncodedFrame::KEY,
requested_bitrate_before_encode);
RecordLatestFrameTimestamps(frame_id,
encoded_frame->reference_time,
encoded_frame->rtp_timestamp);
// Used by chrome/browser/extension/api/cast_streaming/performance_test.cc
TRACE_EVENT_INSTANT1(
"cast_perf_test", "VideoFrameEncoded",
TRACE_EVENT_SCOPE_THREAD,
"rtp_timestamp", encoded_frame->rtp_timestamp);
// At the start of the session, it's important to send reports before each
// frame so that the receiver can properly compute playout times. The reason
// more than one report is sent is because transmission is not guaranteed,
// only best effort, so send enough that one should almost certainly get
// through.
if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
// SendRtcpReport() will schedule future reports to be made if this is the
// last "aggressive report."
++num_aggressive_rtcp_reports_sent_;
const bool is_last_aggressive_report =
(num_aggressive_rtcp_reports_sent_ == kNumAggressiveReportsSentAtStart);
VLOG_IF(1, is_last_aggressive_report) << "Sending last aggressive report.";
SendRtcpReport(is_last_aggressive_report);
}
congestion_control_.SendFrameToTransport(
frame_id, encoded_frame->data.size() * 8, last_send_time_);
if (send_target_playout_delay_) {
encoded_frame->new_playout_delay_ms =
target_playout_delay_.InMilliseconds();
}
transport_sender_->InsertFrame(ssrc_, *encoded_frame);
}
void VideoSender::OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback) {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
base::TimeDelta rtt;
base::TimeDelta avg_rtt;
base::TimeDelta min_rtt;
base::TimeDelta max_rtt;
if (is_rtt_available()) {
rtt = rtt_;
avg_rtt = avg_rtt_;
min_rtt = min_rtt_;
max_rtt = max_rtt_;
congestion_control_.UpdateRtt(rtt);
// Don't use a RTT lower than our average.
rtt = std::max(rtt, avg_rtt);
// Having the RTT values implies the receiver sent back a receiver report
// based on it having received a report from here. Therefore, ensure this
// sender stops aggressively sending reports.
if (num_aggressive_rtcp_reports_sent_ < kNumAggressiveReportsSentAtStart) {
VLOG(1) << "No longer a need to send reports aggressively (sent "
<< num_aggressive_rtcp_reports_sent_ << ").";
num_aggressive_rtcp_reports_sent_ = kNumAggressiveReportsSentAtStart;
ScheduleNextRtcpReport();
}
} else {
// We have no measured value use default.
rtt = base::TimeDelta::FromMilliseconds(kStartRttMs);
}
if (last_send_time_.is_null())
return; // Cannot get an ACK without having first sent a frame.
if (cast_feedback.missing_frames_and_packets.empty()) {
video_encoder_->LatestFrameIdToReference(cast_feedback.ack_frame_id);
// We only count duplicate ACKs when we have sent newer frames.
if (latest_acked_frame_id_ == cast_feedback.ack_frame_id &&
latest_acked_frame_id_ != last_sent_frame_id_) {
duplicate_ack_counter_++;
} else {
duplicate_ack_counter_ = 0;
}
// TODO(miu): The values "2" and "3" should be derived from configuration.
if (duplicate_ack_counter_ >= 2 && duplicate_ack_counter_ % 3 == 2) {
VLOG(1) << "Received duplicate ACK for frame " << latest_acked_frame_id_;
ResendForKickstart();
}
} else {
// Only count duplicated ACKs if there is no NACK request in between.
// This is to avoid aggresive resend.
duplicate_ack_counter_ = 0;
}
base::TimeTicks now = cast_environment_->Clock()->NowTicks();
congestion_control_.AckFrame(cast_feedback.ack_frame_id, now);
cast_environment_->Logging()->InsertFrameEvent(
now,
FRAME_ACK_RECEIVED,
VIDEO_EVENT,
GetRecordedRtpTimestamp(cast_feedback.ack_frame_id),
cast_feedback.ack_frame_id);
const bool is_acked_out_of_order =
static_cast<int32>(cast_feedback.ack_frame_id -
latest_acked_frame_id_) < 0;
VLOG(2) << "Received ACK" << (is_acked_out_of_order ? " out-of-order" : "")
<< " for frame " << cast_feedback.ack_frame_id;
if (!is_acked_out_of_order) {
// Cancel resends of acked frames.
std::vector<uint32> cancel_sending_frames;
while (latest_acked_frame_id_ != cast_feedback.ack_frame_id) {
latest_acked_frame_id_++;
cancel_sending_frames.push_back(latest_acked_frame_id_);
}
transport_sender_->CancelSendingFrames(ssrc_, cancel_sending_frames);
latest_acked_frame_id_ = cast_feedback.ack_frame_id;
}
}
bool VideoSender::ShouldDropNextFrame(base::TimeTicks capture_time) const {
DCHECK(cast_environment_->CurrentlyOn(CastEnvironment::MAIN));
int frames_in_flight = 0;
base::TimeDelta duration_in_flight;
if (!last_send_time_.is_null()) {
frames_in_flight =
static_cast<int32>(last_sent_frame_id_ - latest_acked_frame_id_);
if (frames_in_flight > 0) {
const uint32 oldest_unacked_frame_id = latest_acked_frame_id_ + 1;
duration_in_flight =
capture_time - GetRecordedReferenceTime(oldest_unacked_frame_id);
}
}
frames_in_flight += frames_in_encoder_;
VLOG(2) << frames_in_flight
<< " frames in flight; last sent: " << last_sent_frame_id_
<< "; latest acked: " << latest_acked_frame_id_
<< "; frames in encoder: " << frames_in_encoder_
<< "; duration in flight: "
<< duration_in_flight.InMicroseconds() << " usec ("
<< (target_playout_delay_ > base::TimeDelta() ?
100 * duration_in_flight / target_playout_delay_ :
kint64max) << "%)";
return frames_in_flight >= max_unacked_frames_ ||
duration_in_flight >= target_playout_delay_;
} }
} // namespace cast } // namespace cast
......
media/cast/sender/video_sender.h
View file @ 47581b1f
...@@ -57,36 +57,17 @@ class VideoSender : public FrameSender, ...@@ -57,36 +57,17 @@ class VideoSender : public FrameSender,
const base::TimeTicks& capture_time); const base::TimeTicks& capture_time);
protected: protected:
// Protected for testability. virtual void OnAck(uint32 frame_id) OVERRIDE;
void OnReceivedCastFeedback(const RtcpCastMessage& cast_feedback);
private: private:
// Returns true if there are too many frames in flight, or if the media
// duration of the frames in flight would be too high by sending the next
// frame. The latter metric is determined from the given |capture_time|
// for the next frame to be encoded and sent.
bool ShouldDropNextFrame(base::TimeTicks capture_time) const;
// Called by the |video_encoder_| with the next EncodeFrame to send.
void SendEncodedVideoFrame(int requested_bitrate_before_encode,
scoped_ptr<EncodedFrame> encoded_frame);
// If this value is non zero then a fixed value is used for bitrate.
// If external video encoder is used then bitrate will be fixed to
// (min_bitrate + max_bitrate) / 2.
const size_t fixed_bitrate_;
// Encodes media::VideoFrame images into EncodedFrames. Per configuration, // Encodes media::VideoFrame images into EncodedFrames. Per configuration,
// this will point to either the internal software-based encoder or a proxy to // this will point to either the internal software-based encoder or a proxy to
// a hardware-based encoder. // a hardware-based encoder.
scoped_ptr<VideoEncoder> video_encoder_; scoped_ptr<VideoEncoder> video_encoder_;
// The number of frames currently being processed in |video_encoder_|. // Remember what we set the bitrate to before, no need to set it again if
int frames_in_encoder_; // we get the same value.
uint32 last_bitrate_;
// When we get close to the max number of un-acked frames, we set lower
// the bitrate drastically to ensure that we catch up. Without this we
// risk getting stuck in a catch-up state forever.
CongestionControl congestion_control_;
// NOTE: Weak pointers must be invalidated before all other member variables. // NOTE: Weak pointers must be invalidated before all other member variables.
base::WeakPtrFactory<VideoSender> weak_factory_; base::WeakPtrFactory<VideoSender> weak_factory_;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment