Cast: First stab at implementing adaptive latency

After a bunch of testing, it seems like there is a direct linear
relationship between RTT and the optimal playout delay. Packet loss
is also a factor, but since we expect packet loss to be low in
normal networks, perhaps we can make the formula as easy as RTT * 4.

This is disabled by default, the extension needs to set min_latency_ms
in order to activate it.

BUG=405339

Review URL: https://codereview.chromium.org/562653004

Cr-Commit-Position: refs/heads/master@{#294655}

chrome/common/extensions/api/cast_streaming_rtp_stream.idl

@@ -24,6 +24,9 @@ namespace cast.streaming.rtpStream {
     // playback at the cost of higher latency.
     long maxLatency;
 
+    // Minimum latency in milliseconds. Defaults to |maxLatency|.
+    long? minLatency;
+
     DOMString codecName;
 
     // Synchronization source identifier.

chrome/renderer/extensions/cast_streaming_native_handler.cc

@@ -71,6 +71,8 @@ bool ToCastRtpPayloadParamsOrThrow(v8::Isolate* isolate,
                                    CastRtpPayloadParams* cast_params) {
   cast_params->payload_type = ext_params.payload_type;
   cast_params->max_latency_ms = ext_params.max_latency;
+  cast_params->min_latency_ms =
+      ext_params.min_latency ? *ext_params.min_latency : ext_params.max_latency;
   cast_params->codec_name = ext_params.codec_name;
   cast_params->ssrc = ext_params.ssrc;
   cast_params->feedback_ssrc = ext_params.feedback_ssrc;

chrome/renderer/media/cast_rtp_stream.cc

@@ -160,9 +160,16 @@ bool ToAudioSenderConfig(const CastRtpParams& params,
   config->incoming_feedback_ssrc = params.payload.feedback_ssrc;
   if (config->ssrc == config->incoming_feedback_ssrc)
     return false;
-  config->target_playout_delay =
+  config->min_playout_delay =
+      base::TimeDelta::FromMilliseconds(
+          params.payload.min_latency_ms ?
+          params.payload.min_latency_ms :
+          params.payload.max_latency_ms);
+  config->max_playout_delay =
       base::TimeDelta::FromMilliseconds(params.payload.max_latency_ms);
-  if (config->target_playout_delay <= base::TimeDelta())
+  if (config->min_playout_delay <= base::TimeDelta())
+    return false;
+  if (config->min_playout_delay > config->max_playout_delay)
     return false;
   config->rtp_payload_type = params.payload.payload_type;
   config->use_external_encoder = false;
@@ -188,9 +195,16 @@ bool ToVideoSenderConfig(const CastRtpParams& params,
   config->incoming_feedback_ssrc = params.payload.feedback_ssrc;
   if (config->ssrc == config->incoming_feedback_ssrc)
     return false;
-  config->target_playout_delay =
+  config->min_playout_delay =
+      base::TimeDelta::FromMilliseconds(
+          params.payload.min_latency_ms ?
+          params.payload.min_latency_ms :
+          params.payload.max_latency_ms);
+  config->max_playout_delay =
       base::TimeDelta::FromMilliseconds(params.payload.max_latency_ms);
-  if (config->target_playout_delay <= base::TimeDelta())
+  if (config->min_playout_delay <= base::TimeDelta())
+    return false;
+  if (config->min_playout_delay > config->max_playout_delay)
     return false;
   config->rtp_payload_type = params.payload.payload_type;
   config->width = params.payload.width;
@@ -464,6 +478,7 @@ CastCodecSpecificParams::~CastCodecSpecificParams() {}
 CastRtpPayloadParams::CastRtpPayloadParams()
     : payload_type(0),
       max_latency_ms(0),
+      min_latency_ms(0),
       ssrc(0),
       feedback_ssrc(0),
       clock_rate(0),

chrome/renderer/media/cast_rtp_stream.h

@@ -42,6 +42,10 @@ struct CastRtpPayloadParams {
   // under this threshold.
   int max_latency_ms;
 
+  // Minimum latency.
+  // Default value (0) means use max_latency_ms.
+  int min_latency_ms;
+
   // RTP specific field to identify a stream.
   int ssrc;
 

media/cast/cast_config.cc

@@ -25,7 +25,7 @@ VideoSenderConfig::VideoSenderConfig()
     : ssrc(0),
       incoming_feedback_ssrc(0),
       rtcp_interval(kDefaultRtcpIntervalMs),
-      target_playout_delay(
+      max_playout_delay(
           base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs)),
       rtp_payload_type(0),
       use_external_encoder(false),
@@ -48,7 +48,7 @@ AudioSenderConfig::AudioSenderConfig()
     : ssrc(0),
       incoming_feedback_ssrc(0),
       rtcp_interval(kDefaultRtcpIntervalMs),
-      target_playout_delay(
+      max_playout_delay(
           base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs)),
       rtp_payload_type(0),
       use_external_encoder(false),

media/cast/cast_config.h

@@ -38,12 +38,13 @@ struct AudioSenderConfig {
   int rtcp_interval;
 
   // The total amount of time between a frame's capture/recording on the sender
-  // and its playback on the receiver (i.e., shown to a user).  This is fixed as
-  // a value large enough to give the system sufficient time to encode,
+  // and its playback on the receiver (i.e., shown to a user).  This should be
+  // set to a value large enough to give the system sufficient time to encode,
   // transmit/retransmit, receive, decode, and render; given its run-time
   // environment (sender/receiver hardware performance, network conditions,
   // etc.).
-  base::TimeDelta target_playout_delay;
+  base::TimeDelta min_playout_delay;
+  base::TimeDelta max_playout_delay;
 
   // RTP payload type enum: Specifies the type/encoding of frame data.
   int rtp_payload_type;
@@ -74,12 +75,13 @@ struct VideoSenderConfig {
   int rtcp_interval;
 
   // The total amount of time between a frame's capture/recording on the sender
-  // and its playback on the receiver (i.e., shown to a user).  This is fixed as
-  // a value large enough to give the system sufficient time to encode,
+  // and its playback on the receiver (i.e., shown to a user).  This should be
+  // set to a value large enough to give the system sufficient time to encode,
   // transmit/retransmit, receive, decode, and render; given its run-time
   // environment (sender/receiver hardware performance, network conditions,
   // etc.).
-  base::TimeDelta target_playout_delay;
+  base::TimeDelta min_playout_delay;
+  base::TimeDelta max_playout_delay;
 
   // RTP payload type enum: Specifies the type/encoding of frame data.
   int rtp_payload_type;

media/cast/cast_sender_impl.cc

@@ -133,7 +133,9 @@ void CastSenderImpl::InitializeVideo(
                  weak_factory_.GetWeakPtr(), cast_initialization_cb),
       create_vea_cb,
       create_video_encode_mem_cb,
-      transport_sender_));
+      transport_sender_,
+      base::Bind(&CastSenderImpl::SetTargetPlayoutDelay,
+                 weak_factory_.GetWeakPtr())));
   if (audio_sender_) {
     DCHECK(audio_sender_->GetTargetPlayoutDelay() ==
            video_sender_->GetTargetPlayoutDelay());

media/cast/sender/audio_sender.cc

@@ -33,7 +33,8 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment,
         audio_config.frequency,
         audio_config.ssrc,
         kAudioFrameRate * 2.0, // We lie to increase max outstanding frames.
-        audio_config.target_playout_delay,
+        audio_config.min_playout_delay,
+        audio_config.max_playout_delay,
         NewFixedCongestionControl(audio_config.bitrate)),
       samples_in_encoder_(0),
       weak_factory_(this) {
@@ -61,9 +62,11 @@ AudioSender::AudioSender(scoped_refptr<CastEnvironment> cast_environment,
   transport_config.ssrc = audio_config.ssrc;
   transport_config.feedback_ssrc = audio_config.incoming_feedback_ssrc;
   transport_config.rtp_payload_type = audio_config.rtp_payload_type;
-  // TODO(miu): AudioSender needs to be like VideoSender in providing an upper
-  // limit on the number of in-flight frames.
-  transport_config.stored_frames = max_unacked_frames_;
+  transport_config.stored_frames =
+      std::min(kMaxUnackedFrames,
+               1 + static_cast<int>(max_playout_delay_ *
+                                    max_frame_rate_ /
+                                    base::TimeDelta::FromSeconds(1)));
   transport_config.aes_key = audio_config.aes_key;
   transport_config.aes_iv_mask = audio_config.aes_iv_mask;
 

media/cast/sender/frame_sender.cc

@@ -22,12 +22,16 @@ FrameSender::FrameSender(scoped_refptr<CastEnvironment> cast_environment,
                          int rtp_timebase,
                          uint32 ssrc,
                          double max_frame_rate,
-                         base::TimeDelta playout_delay,
+                         base::TimeDelta min_playout_delay,
+                         base::TimeDelta max_playout_delay,
                          CongestionControl* congestion_control)
     : cast_environment_(cast_environment),
       transport_sender_(transport_sender),
       ssrc_(ssrc),
       rtcp_interval_(rtcp_interval),
+      min_playout_delay_(min_playout_delay == base::TimeDelta() ?
+                         max_playout_delay : min_playout_delay),
+      max_playout_delay_(max_playout_delay),
       max_frame_rate_(max_frame_rate),
       num_aggressive_rtcp_reports_sent_(0),
       last_sent_frame_id_(0),
@@ -40,7 +44,7 @@ FrameSender::FrameSender(scoped_refptr<CastEnvironment> cast_environment,
   DCHECK(transport_sender_);
   DCHECK_GT(rtp_timebase_, 0);
   DCHECK(congestion_control_);
-  SetTargetPlayoutDelay(playout_delay);
+  SetTargetPlayoutDelay(min_playout_delay_);
   send_target_playout_delay_ = false;
   memset(frame_rtp_timestamps_, 0, sizeof(frame_rtp_timestamps_));
 }
@@ -95,6 +99,10 @@ void FrameSender::OnMeasuredRoundTripTime(base::TimeDelta rtt) {
 
 void FrameSender::SetTargetPlayoutDelay(
     base::TimeDelta new_target_playout_delay) {
+  new_target_playout_delay = std::max(new_target_playout_delay,
+                                      min_playout_delay_);
+  new_target_playout_delay = std::min(new_target_playout_delay,
+                                      max_playout_delay_);
   target_playout_delay_ = new_target_playout_delay;
   max_unacked_frames_ =
       std::min(kMaxUnackedFrames,

media/cast/sender/frame_sender.h

@@ -29,7 +29,8 @@ class FrameSender {
               int rtp_timebase,
               uint32 ssrc,
               double max_frame_rate,
-              base::TimeDelta playout_delay,
+              base::TimeDelta min_playout_delay,
+              base::TimeDelta max_playout_delay,
               CongestionControl* congestion_control);
   virtual ~FrameSender();
 
@@ -108,6 +109,8 @@ class FrameSender {
   // environment (sender/receiver hardware performance, network conditions,
   // etc.).
   base::TimeDelta target_playout_delay_;
+  base::TimeDelta min_playout_delay_;
+  base::TimeDelta max_playout_delay_;
 
   // If true, we transmit the target playout delay to the receiver.
   bool send_target_playout_delay_;
@@ -155,6 +158,9 @@ class FrameSender {
   // buffer doesn't overflow.
   scoped_ptr<CongestionControl> congestion_control_;
 
+  // The most recently measured round trip time.
+  base::TimeDelta current_round_trip_time_;
+
  private:
   const bool is_audio_;
 
@@ -164,9 +170,6 @@ class FrameSender {
   base::TimeTicks frame_reference_times_[256];
   RtpTimestamp frame_rtp_timestamps_[256];
 
-  // The most recently measured round trip time.
-  base::TimeDelta current_round_trip_time_;
-
   // NOTE: Weak pointers must be invalidated before all other member variables.
   base::WeakPtrFactory<FrameSender> weak_factory_;
 

media/cast/sender/video_sender.cc

@@ -19,6 +19,16 @@
 namespace media {
 namespace cast {
 
+// The following two constants are used to adjust the target
+// playout delay (when allowed). They were calculated using
+// a combination of cast_benchmark runs and manual testing.
+
+// This is how many round trips we think we need on the network.
+const int kRoundTripsNeeded = 4;
+// This is an estimate of all the the constant time needed
+// independent of network quality.
+const int kConstantTimeMs = 75;
+
 // Note, we use a fixed bitrate value when external video encoder is used.
 // Some hardware encoder shows bad behavior if we set the bitrate too
 // frequently, e.g. quality drop, not abiding by target bitrate, etc.
@@ -29,7 +39,8 @@ VideoSender::VideoSender(
     const CastInitializationCallback& initialization_cb,
     const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
     const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
-    CastTransportSender* const transport_sender)
+    CastTransportSender* const transport_sender,
+    const PlayoutDelayChangeCB& playout_delay_change_cb)
     : FrameSender(
         cast_environment,
         false,
@@ -38,11 +49,13 @@ VideoSender::VideoSender(
         kVideoFrequency,
         video_config.ssrc,
         video_config.max_frame_rate,
-        video_config.target_playout_delay,
+        video_config.min_playout_delay,
+        video_config.max_playout_delay,
         NewFixedCongestionControl(
             (video_config.min_bitrate + video_config.max_bitrate) / 2)),
       frames_in_encoder_(0),
       last_bitrate_(0),
+      playout_delay_change_cb_(playout_delay_change_cb),
       weak_factory_(this) {
   cast_initialization_status_ = STATUS_VIDEO_UNINITIALIZED;
   VLOG(1) << "max_unacked_frames is " << max_unacked_frames_
@@ -82,7 +95,11 @@ VideoSender::VideoSender(
   transport_config.ssrc = video_config.ssrc;
   transport_config.feedback_ssrc = video_config.incoming_feedback_ssrc;
   transport_config.rtp_payload_type = video_config.rtp_payload_type;
-  transport_config.stored_frames = max_unacked_frames_;
+  transport_config.stored_frames =
+      std::min(kMaxUnackedFrames,
+               1 + static_cast<int>(max_playout_delay_ *
+                                    max_frame_rate_ /
+                                    base::TimeDelta::FromSeconds(1)));
   transport_config.aes_key = video_config.aes_key;
   transport_config.aes_iv_mask = video_config.aes_iv_mask;
 
@@ -126,6 +143,14 @@ void VideoSender::InsertRawVideoFrame(
 
   if (ShouldDropNextFrame(capture_time)) {
     VLOG(1) << "Dropping frame due to too many frames currently in-flight.";
+    base::TimeDelta new_target_delay = std::min(
+        current_round_trip_time_ * kRoundTripsNeeded +
+        base::TimeDelta::FromMilliseconds(kConstantTimeMs),
+        max_playout_delay_);
+    if (new_target_delay > target_playout_delay_) {
+      VLOG(1) << "New target delay: " << new_target_delay.InMilliseconds();
+      playout_delay_change_cb_.Run(new_target_delay);
+    }
     return;
   }
 

media/cast/sender/video_sender.h

@@ -25,6 +25,8 @@ namespace cast {
 class CastTransportSender;
 class VideoEncoder;
 
+typedef base::Callback<void(base::TimeDelta)> PlayoutDelayChangeCB;
+
 // Not thread safe. Only called from the main cast thread.
 // This class owns all objects related to sending video, objects that create RTP
 // packets, congestion control, video encoder, parsing and sending of
@@ -40,7 +42,8 @@ class VideoSender : public FrameSender,
               const CastInitializationCallback& initialization_cb,
               const CreateVideoEncodeAcceleratorCallback& create_vea_cb,
               const CreateVideoEncodeMemoryCallback& create_video_encode_mem_cb,
-              CastTransportSender* const transport_sender);
+              CastTransportSender* const transport_sender,
+              const PlayoutDelayChangeCB& playout_delay_change_cb);
 
   virtual ~VideoSender();
 
@@ -79,6 +82,8 @@ class VideoSender : public FrameSender,
   // we get the same value.
   uint32 last_bitrate_;
 
+  PlayoutDelayChangeCB playout_delay_change_cb_;
+
   // NOTE: Weak pointers must be invalidated before all other member variables.
   base::WeakPtrFactory<VideoSender> weak_factory_;
 

media/cast/sender/video_sender_unittest.cc

@@ -111,6 +111,8 @@ class TestPacketSender : public PacketSender {
   DISALLOW_COPY_AND_ASSIGN(TestPacketSender);
 };
 
+void IgnorePlayoutDelayChanges(base::TimeDelta unused_playout_delay) {
+}
 class PeerVideoSender : public VideoSender {
  public:
   PeerVideoSender(
@@ -125,7 +127,8 @@ class PeerVideoSender : public VideoSender {
                     initialization_cb,
                     create_vea_cb,
                     create_video_encode_mem_cb,
-                    transport_sender) {}
+                    transport_sender,
+                    base::Bind(&IgnorePlayoutDelayChanges)) {}
   using VideoSender::OnReceivedCastFeedback;
 };
 }  // namespace

media/cast/test/cast_benchmarks.cc

@@ -225,7 +225,7 @@ class RunOneBenchmark {
                  int max_number_of_video_buffers_used) {
     audio_sender_config_.ssrc = 1;
     audio_sender_config_.incoming_feedback_ssrc = 2;
-    audio_sender_config_.target_playout_delay =
+    audio_sender_config_.max_playout_delay =
         base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs);
     audio_sender_config_.rtp_payload_type = 96;
     audio_sender_config_.use_external_encoder = false;
@@ -247,7 +247,7 @@ class RunOneBenchmark {
 
     video_sender_config_.ssrc = 3;
     video_sender_config_.incoming_feedback_ssrc = 4;
-    video_sender_config_.target_playout_delay =
+    video_sender_config_.max_playout_delay =
         base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs);
     video_sender_config_.rtp_payload_type = 97;
     video_sender_config_.use_external_encoder = false;

media/cast/test/end2end_unittest.cc

@@ -474,7 +474,7 @@ class End2EndTest : public ::testing::Test {
                  int max_number_of_video_buffers_used) {
     audio_sender_config_.ssrc = 1;
     audio_sender_config_.incoming_feedback_ssrc = 2;
-    audio_sender_config_.target_playout_delay =
+    audio_sender_config_.max_playout_delay =
         base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs);
     audio_sender_config_.rtp_payload_type = 96;
     audio_sender_config_.use_external_encoder = false;
@@ -499,7 +499,7 @@ class End2EndTest : public ::testing::Test {
 
     video_sender_config_.ssrc = 3;
     video_sender_config_.incoming_feedback_ssrc = 4;
-    video_sender_config_.target_playout_delay =
+    video_sender_config_.max_playout_delay =
         base::TimeDelta::FromMilliseconds(kTargetPlayoutDelayMs);
     video_sender_config_.rtp_payload_type = 97;
     video_sender_config_.use_external_encoder = false;
@@ -995,9 +995,9 @@ TEST_F(End2EndTest, DISABLED_StartSenderBeforeReceiver) {
 TEST_F(End2EndTest, DropEveryOtherFrame3Buffers) {
   Configure(CODEC_VIDEO_VP8, CODEC_AUDIO_OPUS, kDefaultAudioSamplingRate, 3);
   int target_delay = 300;
-  video_sender_config_.target_playout_delay =
+  video_sender_config_.max_playout_delay =
       base::TimeDelta::FromMilliseconds(target_delay);
-  audio_sender_config_.target_playout_delay =
+  audio_sender_config_.max_playout_delay =
       base::TimeDelta::FromMilliseconds(target_delay);
   video_receiver_config_.rtp_max_delay_ms = target_delay;
   Create();
@@ -1436,6 +1436,12 @@ TEST_F(End2EndTest, OldPacketNetwork) {
 
 TEST_F(End2EndTest, TestSetPlayoutDelay) {
   Configure(CODEC_VIDEO_FAKE, CODEC_AUDIO_PCM16, 32000, 1);
+  video_sender_config_.min_playout_delay =
+      video_sender_config_.max_playout_delay;
+  audio_sender_config_.min_playout_delay =
+      audio_sender_config_.max_playout_delay;
+  video_sender_config_.max_playout_delay = base::TimeDelta::FromSeconds(1);
+  audio_sender_config_.max_playout_delay = base::TimeDelta::FromSeconds(1);
   Create();
   StartBasicPlayer();
   const int kNewDelay = 600;

media/cast/test/sender.cc

@@ -75,7 +75,7 @@ media::cast::AudioSenderConfig GetAudioSenderConfig() {
   audio_config.rtp_payload_type = 127;
   // TODO(miu): The default in cast_defines.h is 100.  Should this be 100, and
   // should receiver.cc's config also be 100?
-  audio_config.target_playout_delay = base::TimeDelta::FromMilliseconds(300);
+  audio_config.max_playout_delay = base::TimeDelta::FromMilliseconds(300);
   return audio_config;
 }
 
@@ -109,7 +109,7 @@ media::cast::VideoSenderConfig GetVideoSenderConfig() {
   video_config.rtp_payload_type = 96;
   // TODO(miu): The default in cast_defines.h is 100.  Should this be 100, and
   // should receiver.cc's config also be 100?
-  video_config.target_playout_delay = base::TimeDelta::FromMilliseconds(300);
+  video_config.max_playout_delay = base::TimeDelta::FromMilliseconds(300);
   return video_config;
 }
 

media/cast/test/simulator.cc

@@ -219,28 +219,28 @@ void RunSimulation(const base::FilePath& source_path,
 
   // Audio sender config.
   AudioSenderConfig audio_sender_config = GetDefaultAudioSenderConfig();
-  audio_sender_config.target_playout_delay =
+  audio_sender_config.max_playout_delay =
       base::TimeDelta::FromMilliseconds(kTargetDelay);
 
   // Audio receiver config.
   FrameReceiverConfig audio_receiver_config =
       GetDefaultAudioReceiverConfig();
   audio_receiver_config.rtp_max_delay_ms =
-      audio_sender_config.target_playout_delay.InMilliseconds();
+      audio_sender_config.max_playout_delay.InMilliseconds();
 
   // Video sender config.
   VideoSenderConfig video_sender_config = GetDefaultVideoSenderConfig();
   video_sender_config.max_bitrate = 2500000;
   video_sender_config.min_bitrate = 2000000;
   video_sender_config.start_bitrate = 2000000;
-  video_sender_config.target_playout_delay =
+  video_sender_config.max_playout_delay =
       base::TimeDelta::FromMilliseconds(kTargetDelay);
 
   // Video receiver config.
   FrameReceiverConfig video_receiver_config =
       GetDefaultVideoReceiverConfig();
   video_receiver_config.rtp_max_delay_ms =
-      video_sender_config.target_playout_delay.InMilliseconds();
+      video_sender_config.max_playout_delay.InMilliseconds();
 
   // Loopback transport.
   LoopBackTransport receiver_to_sender(receiver_env);

media/cast/test/utility/default_config.cc

@@ -62,7 +62,7 @@ AudioSenderConfig GetDefaultAudioSenderConfig() {
   config.channels = recv_config.channels;
   config.bitrate = kDefaultAudioEncoderBitrate;
   config.codec = recv_config.codec;
-  config.target_playout_delay =
+  config.max_playout_delay =
       base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs);
   return config;
 }
@@ -83,7 +83,7 @@ VideoSenderConfig GetDefaultVideoSenderConfig() {
   config.max_number_of_video_buffers_used = 1;
   config.codec = recv_config.codec;
   config.number_of_encode_threads = 2;
-  config.target_playout_delay =
+  config.max_playout_delay =
       base::TimeDelta::FromMilliseconds(kDefaultRtpMaxDelayMs);
   return config;
 }