WebRtcVideoTrackSource: Scale UpdateRect with the frame

Bug: webrtc:11058
Change-Id: I75b68535a33978206acdbb27908167d323822fd4
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1926142
Commit-Queue: Ilya Nikolaevskiy <ilnik@chromium.org>
Auto-Submit: Ilya Nikolaevskiy <ilnik@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#717653}

third_party/blink/renderer/platform/peerconnection/webrtc_video_track_source.cc

@@ -26,6 +26,43 @@ gfx::Rect CropRectangle(const gfx::Rect& input_rect,
   return result;
 }
 
+gfx::Rect ScaleRectangle(const gfx::Rect& input_rect,
+                         gfx::Size original,
+                         gfx::Size scaled) {
+  if (input_rect.IsEmpty()) {
+    return input_rect;
+  }
+  gfx::Rect result;
+  // Rounded down.
+  result.set_x(input_rect.x() * scaled.width() / original.width());
+  result.set_y(input_rect.y() * scaled.height() / original.height());
+  // rounded up.
+  result.set_width(input_rect.width() * scaled.width() / original.width());
+  result.set_height(input_rect.height() * scaled.height() / original.height());
+  // Snap to 2x2 grid because of UV subsampling.
+  if (result.x() % 2) {
+    result.set_x(result.x() - 1);
+    result.set_width(result.width() + 1);
+  }
+  if (result.y() % 2) {
+    result.set_y(result.y() - 1);
+    result.set_height(result.height() + 1);
+  }
+  if (result.width() % 2) {
+    result.set_width(result.width() + 1);
+  }
+  if (result.height() % 2) {
+    result.set_height(result.height() + 1);
+  }
+  // Expand the rect by 2 pixels in each direction, to include any possible
+  // scaling artifacts.
+  result.set_x(result.x() - 2);
+  result.set_y(result.y() - 2);
+  result.set_width(result.width() + 4);
+  result.set_height(result.height() + 4);
+  result.Intersect(gfx::Rect(0, 0, scaled.width(), scaled.height()));
+  return result;
+}
 }  // anonymous namespace
 
 namespace blink {
@@ -181,32 +218,32 @@ void WebRtcVideoTrackSource::OnFrameCaptured(
   if (!video_frame)
     return;
 
-  // If no scaling is needed, return a wrapped version of |frame| directly.
-  // The soft-applied cropping will be taken into account by the remainder
-  // of the pipeline.
-  if (video_frame->natural_size() == video_frame->visible_rect().size()) {
   // The webrtc::VideoFrame::UpdateRect expected by WebRTC must be
   // relative to the |visible_rect()|. We need to translate.
-    base::Optional<gfx::Rect> cropped_rect;
   if (accumulated_update_rect_) {
-      cropped_rect =
+    accumulated_update_rect_ =
         CropRectangle(*accumulated_update_rect_, frame->visible_rect());
   }
-    DeliverFrame(std::move(video_frame), OptionalOrNullptr(cropped_rect),
+
+  // If no scaling is needed, return a wrapped version of |frame| directly.
+  // The soft-applied cropping will be taken into account by the remainder
+  // of the pipeline.
+  if (video_frame->natural_size() == video_frame->visible_rect().size()) {
+    DeliverFrame(std::move(video_frame),
+                 OptionalOrNullptr(accumulated_update_rect_),
                  translated_camera_time_us);
     return;
   }
 
+  if (accumulated_update_rect_) {
+    accumulated_update_rect_ = ScaleRectangle(
+        *accumulated_update_rect_, video_frame->visible_rect().size(),
+        video_frame->natural_size());
+  }
+
   // Delay scaling if |video_frame| is backed by GpuMemoryBuffer.
   if (video_frame->storage_type() ==
       media::VideoFrame::STORAGE_GPU_MEMORY_BUFFER) {
-    // When scaling is applied and any part of the frame has changed, we don't
-    // have reliable changed rect information.
-    if (accumulated_update_rect_.has_value() &&
-        !accumulated_update_rect_->IsEmpty()) {
-      accumulated_update_rect_ = base::nullopt;
-    }
-
     DeliverFrame(std::move(video_frame),
                  OptionalOrNullptr(accumulated_update_rect_),
                  translated_camera_time_us);
@@ -246,12 +283,6 @@ void WebRtcVideoTrackSource::OnFrameCaptured(
                        adapted_size.width(), adapted_size.height(),
                        libyuv::kFilterBilinear);
   }
-  // When scaling is applied and any part of the frame has changed, we don't
-  // have a reliable update rect information.
-  if (accumulated_update_rect_.has_value() &&
-      !accumulated_update_rect_->IsEmpty()) {
-    accumulated_update_rect_ = base::nullopt;
-  }
   DeliverFrame(std::move(scaled_frame),
                OptionalOrNullptr(accumulated_update_rect_),
                translated_camera_time_us);
@@ -291,9 +322,6 @@ void WebRtcVideoTrackSource::DeliverFrame(
     update_rect = nullptr;
   }
 
-  // Clear accumulated_update_rect_.
-  accumulated_update_rect_ = gfx::Rect();
-
   webrtc::VideoFrame::Builder frame_builder =
       webrtc::VideoFrame::Builder()
           .set_video_frame_buffer(
@@ -306,6 +334,9 @@ void WebRtcVideoTrackSource::DeliverFrame(
         update_rect->height()});
   }
   OnFrame(frame_builder.build());
+
+  // Clear accumulated_update_rect_.
+  accumulated_update_rect_ = gfx::Rect();
 }
 
 }  // namespace blink

third_party/blink/renderer/platform/peerconnection/webrtc_video_track_source_test.cc

@@ -358,24 +358,23 @@ TEST_P(WebRtcVideoTrackSourceTest, UpdateRectWithScaling) {
       FrameAdaptation_Scale(kNaturalSize, kScaleToSize));
 
   // Any UPDATE_RECT for the first received frame is expected to get
-  // ignored and the full frame should be marked as updated.
+  // ignored and no update rect should be set.
   const gfx::Rect kUpdateRect1(120, 70, 160, 40);
   EXPECT_CALL(mock_sink_, OnFrame(_))
       .WillOnce(Invoke([](const webrtc::VideoFrame& frame) {
-        ExpectUpdateRectEquals(gfx::Rect(0, 0, frame.width(), frame.height()),
-                               frame.update_rect());
+        EXPECT_FALSE(frame.has_update_rect());
       }));
   int capture_counter = 101;  // arbitrary absolute value
   SendTestFrameWithUpdateRect(kCodedSize, kVisibleRect, kNaturalSize,
                               capture_counter, kUpdateRect1, storage_type);
   Mock::VerifyAndClearExpectations(&mock_sink_);
 
-  // When scaling is applied and UPDATE_RECT is not empty, we always expect a
-  // full update rect.
+  // When scaling is applied and UPDATE_RECT is not empty, we scale the
+  // update rect.
+  // Calculated by hand according to KNaturalSize and KScaleToSize.
   EXPECT_CALL(mock_sink_, OnFrame(_))
       .WillOnce(Invoke([](const webrtc::VideoFrame& frame) {
-        ExpectUpdateRectEquals(gfx::Rect(0, 0, frame.width(), frame.height()),
-                               frame.update_rect());
+        ExpectUpdateRectEquals(gfx::Rect(10, 10, 100, 30), frame.update_rect());
       }));
   SendTestFrameWithUpdateRect(kCodedSize, kVisibleRect, kNaturalSize,
                               ++capture_counter, kUpdateRect1, storage_type);
@@ -390,11 +389,10 @@ TEST_P(WebRtcVideoTrackSourceTest, UpdateRectWithScaling) {
                               ++capture_counter, gfx::Rect(), storage_type);
 
   // When UPDATE_RECT is empty, but the scaling has changed, we expect to
-  // deliver an full UpdateRect.
+  // deliver no known update_rect.
   EXPECT_CALL(mock_sink_, OnFrame(_))
       .WillOnce(Invoke([](const webrtc::VideoFrame& frame) {
-        ExpectUpdateRectEquals(gfx::Rect(0, 0, frame.width(), frame.height()),
-                               frame.update_rect());
+        EXPECT_FALSE(frame.has_update_rect());
       }));
   const gfx::Size kScaleToSize2 = gfx::Size(60, 26);
   track_source_->SetCustomFrameAdaptationParamsForTesting(