[chrome.displaySource][WiFi Display] Video formats capability negotiation

This patch is implementing video formats capability negotiation
based on the given MediaStreamTrack object: finds the video format
having the same frame size and frame rate as the given MediaStreamTrack
among the ones supported by sink device.

BUG=242107

Review URL: https://codereview.chromium.org/1783843002

Cr-Commit-Position: refs/heads/master@{#381692}

extensions/renderer/api/display_source/wifi_display/wifi_display_media_manager.cc

@@ -6,10 +6,28 @@
 
 #include "base/logging.h"
 #include "base/rand_util.h"
+#include "content/public/renderer/media_stream_api.h"
 
 namespace extensions {
 
-WiFiDisplayMediaManager::WiFiDisplayMediaManager() {
+namespace {
+
+const char kErrorNoVideoFormatData[] =
+    "Failed to get video format data from the given MediaStreamTrack object";
+const char kErrorSinkCannotPlayVideo[] =
+    "The sink cannot play video from the given MediaStreamTrack object";
+
+}  // namespace
+
+WiFiDisplayMediaManager::WiFiDisplayMediaManager(
+    const blink::WebMediaStreamTrack& video_track,
+    const blink::WebMediaStreamTrack& audio_track,
+    const ErrorCallback& error_callback)
+  : video_track_(video_track),
+    audio_track_(audio_track),
+    error_callback_(error_callback) {
+  DCHECK(!video_track.isNull() || !audio_track.isNull());
+  DCHECK(!error_callback_.is_null());
 }
 
 WiFiDisplayMediaManager::~WiFiDisplayMediaManager() {
@@ -33,17 +51,22 @@ bool WiFiDisplayMediaManager::IsPaused() const {
 }
 
 wds::SessionType WiFiDisplayMediaManager::GetSessionType() const {
-  NOTIMPLEMENTED();
-  return wds::AudioVideoSession;
+  uint16_t session_type = 0;
+  if (!video_track_.isNull())
+    session_type |= wds::VideoSession;
+
+  if (!audio_track_.isNull())
+    session_type |= wds::AudioSession;
+
+  return static_cast<wds::SessionType>(session_type);
 }
 
 void WiFiDisplayMediaManager::SetSinkRtpPorts(int port1, int port2) {
-  NOTIMPLEMENTED();
+  sink_rtp_ports_ = std::pair<int, int>(port1, port2);
 }
 
-std::pair<int,int> WiFiDisplayMediaManager::GetSinkRtpPorts() const {
-  NOTIMPLEMENTED();
-  return std::pair<int,int>();
+std::pair<int, int> WiFiDisplayMediaManager::GetSinkRtpPorts() const {
+  return sink_rtp_ports_;
 }
 
 int WiFiDisplayMediaManager::GetLocalRtpPort() const {
@@ -51,9 +74,122 @@ int WiFiDisplayMediaManager::GetLocalRtpPort() const {
   return 0;
 }
 
+namespace {
+struct VideoFormat {
+  wds::RateAndResolution rr;
+  int width;
+  int height;
+  int frame_rate;
+};
+
+const VideoFormat cea_table[] = {
+  {wds::CEA640x480p60, 640, 480, 60},
+  {wds::CEA720x480p60, 720, 480, 60},
+  {wds::CEA720x576p50, 720, 576, 50},
+  {wds::CEA1280x720p30, 1280, 720, 30},
+  {wds::CEA1280x720p60, 1280, 720, 60},
+  {wds::CEA1920x1080p30, 1920, 1080, 30},
+  {wds::CEA1920x1080p60, 1920, 1080, 60},
+  {wds::CEA1280x720p25, 1280, 720, 25},
+  {wds::CEA1280x720p50, 1280, 720, 50},
+  {wds::CEA1920x1080p25, 1920, 1080, 25},
+  {wds::CEA1920x1080p50, 1920, 1080, 50},
+  {wds::CEA1280x720p24, 1280, 720, 24},
+  {wds::CEA1920x1080p24, 1920, 1080, 24}
+};
+
+const VideoFormat vesa_table[] = {
+  {wds::VESA800x600p30,    800, 600, 30},
+  {wds::VESA800x600p60,    800, 600, 60},
+  {wds::VESA1024x768p30,   1024, 768, 30},
+  {wds::VESA1024x768p60,   1024, 768, 60},
+  {wds::VESA1152x864p30,   1152, 864, 30},
+  {wds::VESA1152x864p60,   1152, 864, 60},
+  {wds::VESA1280x768p30,   1280, 768, 30},
+  {wds::VESA1280x768p60,   1280, 768, 60},
+  {wds::VESA1280x800p30,   1280, 800, 30},
+  {wds::VESA1280x800p60,   1280, 800, 60},
+  {wds::VESA1360x768p30,   1360, 768, 30},
+  {wds::VESA1360x768p60,   1360, 768, 60},
+  {wds::VESA1366x768p30,   1366, 768, 30},
+  {wds::VESA1366x768p60,   1366, 768, 60},
+  {wds::VESA1280x1024p30,  1280, 1024, 30},
+  {wds::VESA1280x1024p60,  1280, 1024, 60},
+  {wds::VESA1400x1050p30,  1400, 1050, 30},
+  {wds::VESA1400x1050p60,  1400, 1050, 60},
+  {wds::VESA1440x900p30,   1440, 900, 30},
+  {wds::VESA1440x900p60,   1440, 900, 60},
+  {wds::VESA1600x900p30,   1600, 900, 30},
+  {wds::VESA1600x900p60,   1600, 900, 60},
+  {wds::VESA1600x1200p30,  1600, 1200, 30},
+  {wds::VESA1600x1200p60,  1600, 1200, 60},
+  {wds::VESA1680x1024p30,  1680, 1024, 30},
+  {wds::VESA1680x1024p60,  1680, 1024, 60},
+  {wds::VESA1680x1050p30,  1680, 1050, 30},
+  {wds::VESA1680x1050p60,  1680, 1050, 60},
+  {wds::VESA1920x1200p30,  1920, 1200, 30}
+};
+
+const VideoFormat hh_table[] = {
+  {wds::HH800x480p30, 800, 480, 30},
+  {wds::HH800x480p60, 800, 480, 60},
+  {wds::HH854x480p30, 854, 480, 30},
+  {wds::HH854x480p60, 854, 480, 60},
+  {wds::HH864x480p30, 864, 480, 30},
+  {wds::HH864x480p60, 864, 480, 60},
+  {wds::HH640x360p30, 640, 360, 30},
+  {wds::HH640x360p60, 640, 360, 60},
+  {wds::HH960x540p30, 960, 540, 30},
+  {wds::HH960x540p60, 960, 540, 60},
+  {wds::HH848x480p30, 848, 480, 30},
+  {wds::HH848x480p60, 848, 480, 60}
+};
+
+template <wds::ResolutionType type, unsigned N>
+bool FindRateResolution(const media::VideoCaptureFormat* format,
+                        const wds::RateAndResolutionsBitmap& bitmap,
+                        const VideoFormat (&table)[N],
+                        wds::H264VideoFormat* result /*out*/) {
+  for (unsigned i = 0; i < N; ++i) {
+     if (bitmap.test(table[i].rr)) {
+        if (format->frame_size.width() == table[i].width &&
+            format->frame_size.height() == table[i].height &&
+            format->frame_rate == table[i].frame_rate) {
+          result->rate_resolution = table[i].rr;
+          result->type = type;
+          return true;
+        }
+     }
+  }
+  return false;
+}
+
+bool FindOptimalFormat(
+    const media::VideoCaptureFormat* capture_format,
+    const std::vector<wds::H264VideoCodec>& sink_supported_codecs,
+    wds::H264VideoFormat* result /*out*/) {
+  DCHECK(result);
+  for (const wds::H264VideoCodec& codec : sink_supported_codecs) {
+    bool found =
+        FindRateResolution<wds::CEA>(
+            capture_format, codec.cea_rr, cea_table, result) ||
+        FindRateResolution<wds::VESA>(
+            capture_format, codec.vesa_rr, vesa_table, result) ||
+        FindRateResolution<wds::HH>(
+            capture_format, codec.hh_rr, hh_table, result);
+    if (found) {
+      result->profile = codec.profile;
+      result->level = codec.level;
+      return true;
+    }
+  }
+  return false;
+}
+
+}  // namespace
+
 wds::H264VideoFormat WiFiDisplayMediaManager::GetOptimalVideoFormat() const {
-  NOTIMPLEMENTED();
-  return wds::H264VideoFormat();
+  return optimal_video_format_;
 }
 
 void WiFiDisplayMediaManager::SendIDRPicture() {
@@ -67,8 +203,20 @@ std::string WiFiDisplayMediaManager::GetSessionId() const {
 bool WiFiDisplayMediaManager::InitOptimalVideoFormat(
     const wds::NativeVideoFormat& sink_native_format,
     const std::vector<wds::H264VideoCodec>& sink_supported_codecs) {
-  NOTIMPLEMENTED();
+  const media::VideoCaptureFormat* capture_format =
+      content::GetCurrentVideoTrackFormat(video_track_);
+  if (!capture_format) {
+    error_callback_.Run(kErrorNoVideoFormatData);
+    return false;
+  }
+
+  if (!FindOptimalFormat(
+      capture_format, sink_supported_codecs, &optimal_video_format_)) {
+    error_callback_.Run(kErrorSinkCannotPlayVideo);
     return false;
+  }
+
+  return true;
 }
 
 bool WiFiDisplayMediaManager::InitOptimalAudioFormat(

extensions/renderer/api/display_source/wifi_display/wifi_display_media_manager.h

@@ -5,14 +5,26 @@
 #ifndef EXTENSIONS_BROWSER_API_DISPLAY_SOURCE_WIFI_DISPLAY_WIFI_DISPLAY_MEDIA_MANAGER_H_
 #define EXTENSIONS_BROWSER_API_DISPLAY_SOURCE_WIFI_DISPLAY_WIFI_DISPLAY_MEDIA_MANAGER_H_
 
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "base/callback.h"
 #include "base/macros.h"
+#include "third_party/WebKit/public/web/WebDOMMediaStreamTrack.h"
 #include "third_party/wds/src/libwds/public/media_manager.h"
 
 namespace extensions {
 
 class WiFiDisplayMediaManager : public wds::SourceMediaManager {
  public:
-  WiFiDisplayMediaManager();
+  using ErrorCallback = base::Callback<void(const std::string&)>;
+
+  WiFiDisplayMediaManager(
+      const blink::WebMediaStreamTrack& video_track,
+      const blink::WebMediaStreamTrack& audio_track,
+      const ErrorCallback& error_callback);
+
   ~WiFiDisplayMediaManager() override;
 
  private:
@@ -24,7 +36,7 @@ class WiFiDisplayMediaManager : public wds::SourceMediaManager {
   bool IsPaused() const override;
   wds::SessionType GetSessionType() const override;
   void SetSinkRtpPorts(int port1, int port2) override;
-  std::pair<int,int> GetSinkRtpPorts() const override;
+  std::pair<int, int> GetSinkRtpPorts() const override;
   int GetLocalRtpPort() const override;
 
   bool InitOptimalVideoFormat(
@@ -39,6 +51,15 @@ class WiFiDisplayMediaManager : public wds::SourceMediaManager {
 
   std::string GetSessionId() const override;
 
+ private:
+  blink::WebMediaStreamTrack video_track_;
+  blink::WebMediaStreamTrack audio_track_;
+
+  std::pair<int, int> sink_rtp_ports_;
+  wds::H264VideoFormat optimal_video_format_;
+
+  ErrorCallback error_callback_;
+
   DISALLOW_COPY_AND_ASSIGN(WiFiDisplayMediaManager);
 };
 

extensions/renderer/api/display_source/wifi_display/wifi_display_session.cc

@@ -4,6 +4,8 @@
 
 #include "extensions/renderer/api/display_source/wifi_display/wifi_display_session.h"
 
+#include <utility>
+
 #include "base/logging.h"
 #include "base/timer/timer.h"
 #include "content/public/common/service_registry.h"
@@ -69,15 +71,20 @@ void WiFiDisplaySession::Start(const CompletionCallback& callback) {
 
 void WiFiDisplaySession::Terminate(const CompletionCallback& callback) {
   DCHECK_EQ(DisplaySourceSession::Established, state_);
-  service_->Disconnect();
-  state_ = DisplaySourceSession::Terminating;
+  Terminate();
   teminate_completion_callback_ = callback;
 }
 
 void WiFiDisplaySession::OnConnected(const mojo::String& ip_address) {
   DCHECK_EQ(DisplaySourceSession::Established, state_);
   ip_address_ = ip_address;
-  media_manager_.reset(new WiFiDisplayMediaManager());
+  media_manager_.reset(
+      new WiFiDisplayMediaManager(
+          params_.video_track,
+          params_.audio_track,
+          base::Bind(
+              &WiFiDisplaySession::OnMediaError,
+              weak_factory_.GetWeakPtr())));
   wfd_source_.reset(wds::Source::Create(this, media_manager_.get(), this));
   wfd_source_->Start();
 }
@@ -117,26 +124,6 @@ void WiFiDisplaySession::OnMessage(const mojo::String& data) {
   wfd_source_->RTSPDataReceived(data);
 }
 
-void WiFiDisplaySession::OnIPCConnectionError() {
-  // We must explicitly notify the session termination as it will never
-  // arrive from browser process (IPC is broken).
-  switch (state_) {
-    case DisplaySourceSession::Idle:
-    case DisplaySourceSession::Establishing:
-      RunStartCallback(false, kErrorInternal);
-      break;
-    case DisplaySourceSession::Terminating:
-    case DisplaySourceSession::Established:
-      error_callback_.Run(api::display_source::ERROR_TYPE_UNKNOWN_ERROR,
-                          kErrorInternal);
-      state_ = DisplaySourceSession::Idle;
-      terminated_callback_.Run();
-      break;
-    default:
-      NOTREACHED();
-  }
-}
-
 std::string WiFiDisplaySession::GetLocalIPAddress() const {
   return ip_address_;
 }
@@ -178,18 +165,45 @@ void WiFiDisplaySession::ErrorOccurred(wds::ErrorType error) {
     error_callback_.Run(api::display_source::ERROR_TYPE_UNKNOWN_ERROR,
                         kErrorInternal);
   }
-
-  if (state_ == DisplaySourceSession::Established) {
   // The session cannot continue.
-    service_->Disconnect();
-    state_ = DisplaySourceSession::Terminating;
-  }
+  Terminate();
 }
 
 void WiFiDisplaySession::SessionCompleted() {
   DCHECK_NE(DisplaySourceSession::Idle, state_);
-  if (state_ == DisplaySourceSession::Established) {
   // The session has finished normally.
+  Terminate();
+}
+
+void WiFiDisplaySession::OnIPCConnectionError() {
+  // We must explicitly notify the session termination as it will never
+  // arrive from browser process (IPC is broken).
+  switch (state_) {
+    case DisplaySourceSession::Idle:
+    case DisplaySourceSession::Establishing:
+      RunStartCallback(false, kErrorInternal);
+      break;
+    case DisplaySourceSession::Terminating:
+    case DisplaySourceSession::Established:
+      error_callback_.Run(api::display_source::ERROR_TYPE_UNKNOWN_ERROR,
+                          kErrorInternal);
+      state_ = DisplaySourceSession::Idle;
+      terminated_callback_.Run();
+      break;
+    default:
+      NOTREACHED();
+  }
+}
+
+void WiFiDisplaySession::OnMediaError(const std::string& error) {
+  DCHECK_NE(DisplaySourceSession::Idle, state_);
+  error_callback_.Run(api::display_source::ERROR_TYPE_MEDIA_PIPELINE_ERROR,
+                      error);
+  Terminate();
+}
+
+void WiFiDisplaySession::Terminate() {
+  if (state_ == DisplaySourceSession::Established) {
     service_->Disconnect();
     state_ = DisplaySourceSession::Terminating;
   }

extensions/renderer/api/display_source/wifi_display/wifi_display_session.h

@@ -63,6 +63,11 @@ class WiFiDisplaySession: public DisplaySourceSession,
   // A connection error handler for the mojo objects used in this class.
   void OnIPCConnectionError();
 
+  // An error handler for media pipeline error.
+  void OnMediaError(const std::string& error);
+
+  void Terminate();
+
   void RunStartCallback(bool success, const std::string& error = "");
   void RunTerminateCallback(bool success, const std::string& error = "");