Support getCapabilities() for audio and video sources other than getUserMedia()

This CL adds support for audio and video properties in MediaStreamTrack.getCapabilities()
for audio and video sources other than getUserMedia().

Bug: 293292
Change-Id: I14569ad829b80ade60ea52f371edc0bffd282583
Reviewed-on: https://chromium-review.googlesource.com/934481
Commit-Queue: Chandan Padhi <c.padhi@samsung.com>
Reviewed-by: Avi Drissman <avi@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Reviewed-by: Emircan Uysaler <emircan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#539804}

content/public/renderer/media_stream_utils.cc

@@ -12,6 +12,7 @@
 #include "base/rand_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "content/renderer/media/stream/external_media_stream_audio_source.h"
+#include "content/renderer/media/stream/media_stream_constraints_util.h"
 #include "content/renderer/media/stream/media_stream_video_capturer_source.h"
 #include "content/renderer/media/stream/media_stream_video_source.h"
 #include "content/renderer/media/stream/media_stream_video_track.h"
@@ -33,16 +34,22 @@ bool AddVideoTrackToMediaStream(
     return false;
   }
 
-  blink::WebMediaStreamSource web_media_stream_source;
+  media::VideoCaptureFormats preferred_formats =
+      video_source->GetPreferredFormats();
   MediaStreamVideoSource* const media_stream_source =
       new MediaStreamVideoCapturerSource(
           MediaStreamSource::SourceStoppedCallback(), std::move(video_source));
   const blink::WebString track_id =
       blink::WebString::FromUTF8(base::GenerateGUID());
+  blink::WebMediaStreamSource web_media_stream_source;
   web_media_stream_source.Initialize(
       track_id, blink::WebMediaStreamSource::kTypeVideo, track_id, is_remote);
   // Takes ownership of |media_stream_source|.
   web_media_stream_source.SetExtraData(media_stream_source);
+  web_media_stream_source.SetCapabilities(ComputeCapabilitiesForVideoSource(
+      track_id, preferred_formats,
+      media::VideoFacingMode::MEDIA_VIDEO_FACING_NONE,
+      false /* is_device_capture */));
   web_media_stream->AddTrack(MediaStreamVideoTrack::CreateVideoTrack(
       media_stream_source, MediaStreamVideoSource::ConstraintsCallback(),
       true));
@@ -82,6 +89,11 @@ bool AddAudioTrackToMediaStream(
   // Takes ownership of |media_stream_source|.
   web_media_stream_source.SetExtraData(media_stream_source);
 
+  blink::WebMediaStreamSource::Capabilities capabilities;
+  capabilities.device_id = track_id;
+  capabilities.echo_cancellation = std::vector<bool>({false});
+  web_media_stream_source.SetCapabilities(capabilities);
+
   blink::WebMediaStreamTrack web_media_stream_track;
   web_media_stream_track.Initialize(web_media_stream_source);
   if (!media_stream_source->ConnectToTrack(web_media_stream_track))

content/renderer/media/stream/media_stream_center.cc

@@ -51,6 +51,11 @@ void CreateNativeAudioMediaStreamTrack(
     DVLOG(1) << "Creating WebAudio media stream source.";
     media_stream_source = new WebAudioMediaStreamSource(&source);
     source.SetExtraData(media_stream_source);  // Takes ownership.
+
+    blink::WebMediaStreamSource::Capabilities capabilities;
+    capabilities.device_id = source.Id();
+    capabilities.echo_cancellation = std::vector<bool>({false});
+    source.SetCapabilities(capabilities);
   }
 
   if (media_stream_source)

content/renderer/media/stream/media_stream_constraints_util.cc

@@ -10,13 +10,17 @@
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/utf_string_conversions.h"
 #include "content/renderer/media/stream/media_stream_constraints_util_sets.h"
-#include "third_party/WebKit/public/platform/WebMediaConstraints.h"
+#include "content/renderer/media/stream/media_stream_constraints_util_video_device.h"
 #include "third_party/WebKit/public/platform/WebString.h"
 
 namespace content {
 
 namespace {
 
+// TODO(c.padhi): Allow frame rates lower than 1Hz,
+// see https://crbug.com/814131.
+const float kMinDeviceCaptureFrameRate = 1.0f;
+
 template <typename P, typename T>
 bool ScanConstraintsForExactValue(const blink::WebMediaConstraints& constraints,
                                   P picker,
@@ -319,4 +323,33 @@ double StringConstraintFitnessDistance(
   return 1.0;
 }
 
+blink::WebMediaStreamSource::Capabilities ComputeCapabilitiesForVideoSource(
+    const blink::WebString& device_id,
+    const media::VideoCaptureFormats& formats,
+    media::VideoFacingMode facing_mode,
+    bool is_device_capture) {
+  blink::WebMediaStreamSource::Capabilities capabilities;
+  capabilities.device_id = device_id;
+  if (is_device_capture)
+    capabilities.facing_mode = ToWebFacingMode(facing_mode);
+  if (!formats.empty()) {
+    int max_width = 1;
+    int max_height = 1;
+    float min_frame_rate =
+        is_device_capture ? kMinDeviceCaptureFrameRate : 0.0f;
+    float max_frame_rate = min_frame_rate;
+    for (const auto& format : formats) {
+      max_width = std::max(max_width, format.frame_size.width());
+      max_height = std::max(max_height, format.frame_size.height());
+      max_frame_rate = std::max(max_frame_rate, format.frame_rate);
+    }
+    capabilities.width = {1, max_width};
+    capabilities.height = {1, max_height};
+    capabilities.aspect_ratio = {1.0 / max_height,
+                                 static_cast<double>(max_width)};
+    capabilities.frame_rate = {min_frame_rate, max_frame_rate};
+  }
+  return capabilities;
+}
+
 }  // namespace content

content/renderer/media/stream/media_stream_constraints_util.h

@@ -11,8 +11,10 @@
 #include "content/common/content_export.h"
 #include "content/renderer/media/stream/media_stream_audio_processor_options.h"
 #include "content/renderer/media/stream/video_track_adapter.h"
+#include "media/base/video_facing.h"
 #include "media/capture/video_capture_types.h"
 #include "third_party/WebKit/public/platform/WebMediaConstraints.h"
+#include "third_party/WebKit/public/platform/WebMediaStreamSource.h"
 #include "third_party/WebKit/public/platform/modules/mediastream/media_devices.mojom.h"
 #include "third_party/webrtc/api/optional.h"
 
@@ -368,6 +370,14 @@ double StringConstraintFitnessDistance(
     const blink::WebString& value,
     const blink::StringConstraint& constraint);
 
+// This method computes capabilities for a video source based on the given
+// |formats|. |facing_mode| is valid only in case of video device capture.
+blink::WebMediaStreamSource::Capabilities CONTENT_EXPORT
+ComputeCapabilitiesForVideoSource(const blink::WebString& device_id,
+                                  const media::VideoCaptureFormats& formats,
+                                  media::VideoFacingMode facing_mode,
+                                  bool is_device_capture);
+
 }  // namespace content
 
 #endif  // CONTENT_RENDERER_MEDIA_STREAM_MEDIA_STREAM_CONSTRAINTS_UTIL_H_

content/renderer/media/stream/remote_media_stream_track_adapter.cc

@@ -41,6 +41,11 @@ void RemoteVideoTrackAdapter::InitializeWebVideoTrack(
       new MediaStreamRemoteVideoSource(std::move(observer)));
   InitializeWebTrack(blink::WebMediaStreamSource::kTypeVideo);
   web_track()->Source().SetExtraData(video_source.get());
+
+  blink::WebMediaStreamSource::Capabilities capabilities;
+  capabilities.device_id = blink::WebString::FromUTF8(id());
+  web_track()->Source().SetCapabilities(capabilities);
+
   MediaStreamVideoTrack* media_stream_track = new MediaStreamVideoTrack(
       video_source.release(), MediaStreamVideoSource::ConstraintsCallback(),
       enabled);
@@ -83,6 +88,12 @@ void RemoteAudioTrackAdapter::InitializeWebAudioTrack() {
   MediaStreamAudioSource* const source =
       new PeerConnectionRemoteAudioSource(observed_track().get());
   web_track()->Source().SetExtraData(source);  // Takes ownership.
+
+  blink::WebMediaStreamSource::Capabilities capabilities;
+  capabilities.device_id = blink::WebString::FromUTF8(id());
+  capabilities.echo_cancellation = std::vector<bool>({false});
+  web_track()->Source().SetCapabilities(capabilities);
+
   source->ConnectToTrack(*(web_track()));
 }
 

content/renderer/media/stream/user_media_processor.cc

@@ -47,10 +47,6 @@
 namespace content {
 namespace {
 
-// TODO(c.padhi): Allow frame rates lower than 1Hz,
-// see https://crbug.com/814131.
-const float kMinDeviceCaptureFrameRate = 1.0f;
-
 void CopyFirstString(const blink::StringConstraint& constraint,
                      std::string* destination) {
   if (!constraint.Exact().IsEmpty())
@@ -110,31 +106,6 @@ void SurfaceHardwareEchoCancellationSetting(
     source->SetEchoCancellation(true);
 }
 
-blink::WebMediaStreamSource::Capabilities ComputeCapabilities(
-    const MediaStreamDevice& device,
-    const media::VideoCaptureFormats& formats,
-    bool is_device_capture) {
-  int max_width = 1;
-  int max_height = 1;
-  float min_frame_rate = is_device_capture ? kMinDeviceCaptureFrameRate : 0.0f;
-  float max_frame_rate = min_frame_rate;
-  for (const auto& format : formats) {
-    max_width = std::max(max_width, format.frame_size.width());
-    max_height = std::max(max_height, format.frame_size.height());
-    max_frame_rate = std::max(max_frame_rate, format.frame_rate);
-  }
-  blink::WebMediaStreamSource::Capabilities capabilities;
-  capabilities.device_id = blink::WebString::FromUTF8(device.id);
-  capabilities.width = {1, max_width};
-  capabilities.height = {1, max_height};
-  capabilities.aspect_ratio = {1.0 / max_height,
-                               static_cast<double>(max_width)};
-  capabilities.frame_rate = {min_frame_rate, max_frame_rate};
-  if (is_device_capture)
-    capabilities.facing_mode = ToWebFacingMode(device.video_facing);
-  return capabilities;
-}
-
 }  // namespace
 
 UserMediaRequest::UserMediaRequest(
@@ -812,8 +783,9 @@ blink::WebMediaStreamSource UserMediaProcessor::InitializeVideoSourceObject(
     source.SetExtraData(CreateVideoSource(
         device, base::Bind(&UserMediaProcessor::OnLocalSourceStopped,
                            weak_factory_.GetWeakPtr())));
-    source.SetCapabilities(ComputeCapabilities(
-        device, *current_request_info_->GetVideoFormats(device.id),
+    source.SetCapabilities(ComputeCapabilitiesForVideoSource(
+        blink::WebString::FromUTF8(device.id),
+        *current_request_info_->GetVideoFormats(device.id), device.video_facing,
         current_request_info_->is_video_device_capture()));
     local_sources_.push_back(source);
   }

content/renderer/media_capture_from_element/canvas_capture_handler.cc

@@ -13,6 +13,7 @@
 #include "base/strings/utf_string_conversions.h"
 #include "components/viz/common/gl_helper.h"
 #include "content/public/renderer/render_thread.h"
+#include "content/renderer/media/stream/media_stream_constraints_util.h"
 #include "content/renderer/media/stream/media_stream_video_capturer_source.h"
 #include "content/renderer/media/stream/media_stream_video_source.h"
 #include "content/renderer/media/stream/media_stream_video_track.h"
@@ -461,13 +462,18 @@ void CanvasCaptureHandler::AddVideoCapturerSourceToVideoTrack(
   std::string str_track_id;
   base::Base64Encode(base::RandBytesAsString(64), &str_track_id);
   const blink::WebString track_id = blink::WebString::FromASCII(str_track_id);
-  blink::WebMediaStreamSource webkit_source;
+  media::VideoCaptureFormats preferred_formats = source->GetPreferredFormats();
   std::unique_ptr<MediaStreamVideoSource> media_stream_source(
       new MediaStreamVideoCapturerSource(
           MediaStreamSource::SourceStoppedCallback(), std::move(source)));
+  blink::WebMediaStreamSource webkit_source;
   webkit_source.Initialize(track_id, blink::WebMediaStreamSource::kTypeVideo,
                            track_id, false);
   webkit_source.SetExtraData(media_stream_source.get());
+  webkit_source.SetCapabilities(ComputeCapabilitiesForVideoSource(
+      track_id, preferred_formats,
+      media::VideoFacingMode::MEDIA_VIDEO_FACING_NONE,
+      false /* is_device_capture */));
 
   web_track->Initialize(webkit_source);
   web_track->SetTrackData(new MediaStreamVideoTrack(

content/renderer/renderer_blink_platform_impl.cc

@@ -1025,6 +1025,11 @@ void RendererBlinkPlatformImpl::CreateHTMLAudioElementCapturer(
   // Takes ownership of |media_stream_source|.
   web_media_stream_source.SetExtraData(media_stream_source);
 
+  blink::WebMediaStreamSource::Capabilities capabilities;
+  capabilities.device_id = track_id;
+  capabilities.echo_cancellation = std::vector<bool>({false});
+  web_media_stream_source.SetCapabilities(capabilities);
+
   media_stream_source->ConnectToTrack(web_media_stream_track);
   web_media_stream->AddTrack(web_media_stream_track);
 #endif

third_party/WebKit/LayoutTests/fast/mediastream/MediaStreamTrack-getCapabilities.html

@@ -19,31 +19,111 @@ promise_test(function() {
     var capabilities = stream.getVideoTracks()[0].getCapabilities();
     assert_greater_than(Object.keys(capabilities).length, 0);
     assert_true(capabilities.hasOwnProperty('deviceId'));
-    if (capabilities.hasOwnProperty('width')) {
+    assert_true(capabilities.hasOwnProperty('facingMode'));
+    verifyVideoRangeProperties(capabilities);
+  });
+}, 'getCapabilities() support for getUserMedia() video track.');
+
+test(function() {
+  var stream = (new AudioContext()).createMediaStreamDestination().stream;
+  var capabilities = stream.getAudioTracks()[0].getCapabilities();
+  assert_equals(Object.keys(capabilities).length, 2);
+  assert_true(capabilities.hasOwnProperty('deviceId'));
+  assert_true(capabilities.hasOwnProperty('echoCancellation'));
+  assert_equals(Object.keys(capabilities.echoCancellation).length, 1);
+  assert_equals(capabilities.echoCancellation[0], false);
+}, 'getCapabilities() support for audio track associated with a MediaStreamAudioDestinationNode.');
+
+test(function() {
+  var canvas = document.createElement('canvas');
+  var stream = canvas.captureStream();
+  var capabilities = stream.getVideoTracks()[0].getCapabilities();
+  assert_greater_than(Object.keys(capabilities).length, 0);
+  assert_true(capabilities.hasOwnProperty('deviceId'));
+  assert_true(capabilities.hasOwnProperty('facingMode'));
+  assert_equals(Object.keys(capabilities.facingMode).length, 0);
+  verifyVideoRangeProperties(capabilities);
+}, 'getCapabilities() support for video track associated with a canvas element.');
+
+test(function() {
+  var video = document.createElement('video');
+  video.src = "../../external/wpt/media/test-av-384k-44100Hz-1ch-320x240-30fps-10kfr.webm";
+  video.play();
+  video.oncanplay = function() {
+    var stream = video.captureStream();
+    var audioCapabilities = stream.getAudioTracks()[0].getCapabilities();
+    var videoCapabilities = stream.getVideoTracks()[0].getCapabilities();
+    assert_equals(Object.keys(audioCapabilities).length, 2);
+    assert_true(audioCapabilities.hasOwnProperty('deviceId'));
+    assert_true(audioCapabilities.hasOwnProperty('echoCancellation'));
+    assert_equals(Object.keys(audioCapabilities.echoCancellation).length, 1);
+    assert_equals(audioCapabilities.echoCancellation[0], false);
+    assert_greater_than(Object.keys(videoCapabilities).length, 0);
+    assert_true(videoCapabilities.hasOwnProperty('deviceId'));
+    assert_true(videoCapabilities.hasOwnProperty('facingMode'));
+    assert_equals(Object.keys(videoCapabilities.facingMode).length, 0);
+    verifyVideoRangeProperties(videoCapabilities);
+  };
+}, 'getCapabilities() support for audio and video tracks associated with a video element.');
+
+promise_test(function() {
+  var caller = new RTCPeerConnection();
+  var callee = new RTCPeerConnection();
+  return navigator.mediaDevices.getUserMedia({audio:true, video:true})
+  .then((stream) => {
+    caller.addStream(stream);
+    return createAndSetOffer(caller, callee);
+  }).then(() => {
+    var remoteStream = callee.getRemoteStreams()[0];
+    var audioCapabilities = remoteStream.getAudioTracks()[0].getCapabilities();
+    var videoCapabilities = remoteStream.getVideoTracks()[0].getCapabilities();
+    assert_equals(Object.keys(audioCapabilities).length, 2);
+    assert_true(audioCapabilities.hasOwnProperty('deviceId'));
+    assert_true(audioCapabilities.hasOwnProperty('echoCancellation'));
+    assert_equals(Object.keys(audioCapabilities.echoCancellation).length, 1);
+    assert_equals(audioCapabilities.echoCancellation[0], false);
+    assert_greater_than(Object.keys(videoCapabilities).length, 0);
+    assert_true(videoCapabilities.hasOwnProperty('deviceId'));
+    assert_true(videoCapabilities.hasOwnProperty('facingMode'));
+    assert_equals(Object.keys(videoCapabilities.facingMode).length, 0);
+  });
+}, 'getCapabilities() support for audio and video tracks associated with a RTCPeerConnection.');
+
+function createAndSetOffer(caller, callee) {
+  var description = null;
+  return caller.createOffer()
+  .then(offer => {
+    description = offer;
+    return caller.setLocalDescription(description);
+  }).then(() => {
+    return callee.setRemoteDescription(description);
+  });
+}
+
+function verifyVideoRangeProperties(capabilities) {
+  if (capabilities.hasOwnProperty('width')) {
       assert_equals(Object.keys(capabilities.width).length, 2);
       assert_true(capabilities.width.hasOwnProperty('min'));
       assert_true(capabilities.width.hasOwnProperty('max'));
       assert_less_than_equal(capabilities.width.min, capabilities.width.max);
-    }
-    if (capabilities.hasOwnProperty('height')) {
-      assert_equals(Object.keys(capabilities.height).length, 2);
-      assert_true(capabilities.height.hasOwnProperty('min'));
-      assert_true(capabilities.height.hasOwnProperty('max'));
-      assert_less_than_equal(capabilities.height.min, capabilities.height.max);
-    }
-    if (capabilities.hasOwnProperty('aspectRatio')) {
-      assert_equals(Object.keys(capabilities.aspectRatio).length, 2);
-      assert_true(capabilities.aspectRatio.hasOwnProperty('min'));
-      assert_true(capabilities.aspectRatio.hasOwnProperty('max'));
-      assert_less_than_equal(capabilities.aspectRatio.min, capabilities.aspectRatio.max);
-    }
-    if (capabilities.hasOwnProperty('frameRate')) {
-      assert_equals(Object.keys(capabilities.frameRate).length, 2);
-      assert_true(capabilities.frameRate.hasOwnProperty('min'));
-      assert_true(capabilities.frameRate.hasOwnProperty('max'));
-      assert_less_than_equal(capabilities.frameRate.min, capabilities.frameRate.max);
-    }
-    assert_true(capabilities.hasOwnProperty('facingMode'));
-  });
-}, 'getCapabilities() support for getUserMedia() video track.');
+  }
+  if (capabilities.hasOwnProperty('height')) {
+    assert_equals(Object.keys(capabilities.height).length, 2);
+    assert_true(capabilities.height.hasOwnProperty('min'));
+    assert_true(capabilities.height.hasOwnProperty('max'));
+    assert_less_than_equal(capabilities.height.min, capabilities.height.max);
+  }
+  if (capabilities.hasOwnProperty('aspectRatio')) {
+    assert_equals(Object.keys(capabilities.aspectRatio).length, 2);
+    assert_true(capabilities.aspectRatio.hasOwnProperty('min'));
+    assert_true(capabilities.aspectRatio.hasOwnProperty('max'));
+    assert_less_than_equal(capabilities.aspectRatio.min, capabilities.aspectRatio.max);
+  }
+  if (capabilities.hasOwnProperty('frameRate')) {
+    assert_equals(Object.keys(capabilities.frameRate).length, 2);
+    assert_true(capabilities.frameRate.hasOwnProperty('min'));
+    assert_true(capabilities.frameRate.hasOwnProperty('max'));
+    assert_less_than_equal(capabilities.frameRate.min, capabilities.frameRate.max);
+  }
+}
+</script>
\ No newline at end of file
-</script>