[WebXR] Don't show "unresponsive" page dialog when page is throttled

A page may not be submitting frames because blink isn't allowing it to
submit frames based on various states. In these cases, the browser
should not be showing UI indicating that the page is unresponsive.
This change adds plumbing for blink to inform the browser process that
it is the one throttling frame requests so that the browser can suspend
any timeout logic, and potentially (in the future) take over rendering
a more customized message.

Bug: 1009813
Change-Id: I2d1a3bb609eb305225493b67c630b32999735a6b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1841585Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Klaus Weidner <klausw@chromium.org>
Commit-Queue: Alexander Cooper <alcooper@chromium.org>
Cr-Commit-Position: refs/heads/master@{#704461}

chrome/browser/vr/service/browser_xr_runtime.cc

@@ -444,6 +444,15 @@ void BrowserXRRuntime::ExitPresent(VRServiceImpl* service) {
   }
 }
 
+void BrowserXRRuntime::SetFramesThrottled(const VRServiceImpl* service,
+                                          bool throttled) {
+  if (service == presenting_service_) {
+    for (BrowserXRRuntimeObserver& observer : observers_) {
+      observer.SetFramesThrottled(throttled);
+    }
+  }
+}
+
 void BrowserXRRuntime::RequestSession(
     VRServiceImpl* service,
     const device::mojom::XRRuntimeSessionOptionsPtr& options,

chrome/browser/vr/service/browser_xr_runtime.h

@@ -38,6 +38,8 @@ class BrowserXRRuntimeObserver : public base::CheckedObserver {
   // session. There can only be at most one active immersive session for the
   // XRRuntime. Set to null when there is no active immersive session.
   virtual void SetWebXRWebContents(content::WebContents* contents) = 0;
+
+  virtual void SetFramesThrottled(bool throttled) = 0;
 };
 
 // This class wraps a physical device's interfaces, and registers for events.
@@ -67,6 +69,7 @@ class BrowserXRRuntime : public device::mojom::XRRuntimeEventListener {
   void OnServiceAdded(VRServiceImpl* service);
   void OnServiceRemoved(VRServiceImpl* service);
   void ExitPresent(VRServiceImpl* service);
+  void SetFramesThrottled(const VRServiceImpl* service, bool throttled);
   void RequestSession(VRServiceImpl* service,
                       const device::mojom::XRRuntimeSessionOptionsPtr& options,
                       RequestSessionCallback callback);

chrome/browser/vr/service/vr_service_impl.cc

@@ -542,6 +542,17 @@ void VRServiceImpl::ExitPresent() {
     immersive_runtime->ExitPresent(this);
 }
 
+void VRServiceImpl::SetFramesThrottled(bool throttled) {
+  if (throttled != frames_throttled_) {
+    frames_throttled_ = throttled;
+    BrowserXRRuntime* immersive_runtime =
+        runtime_manager_->GetImmersiveRuntime();
+    if (immersive_runtime) {
+      immersive_runtime->SetFramesThrottled(this, frames_throttled_);
+    }
+  }
+}
+
 void VRServiceImpl::SetListeningForActivate(
     mojo::PendingRemote<device::mojom::VRDisplayClient> display_client) {
   // TODO(crbug.com/999745): Remove the check if the condition to check if

chrome/browser/vr/service/vr_service_impl.h

@@ -65,6 +65,7 @@ class VR_EXPORT VRServiceImpl : public device::mojom::VRService,
       device::mojom::XRSessionOptionsPtr options,
       device::mojom::VRService::SupportsSessionCallback callback) override;
   void ExitPresent() override;
+  void SetFramesThrottled(bool throttled) override;
   // device::mojom::VRService WebVR compatibility functions
   void GetImmersiveVRDisplayInfo(
       device::mojom::VRService::GetImmersiveVRDisplayInfoCallback callback)
@@ -168,6 +169,7 @@ class VR_EXPORT VRServiceImpl : public device::mojom::VRService,
 
   bool initialization_complete_ = false;
   bool in_focused_frame_ = false;
+  bool frames_throttled_ = false;
 
   std::map<device::mojom::XRDeviceId, XrConsentPromptLevel>
       consent_granted_devices_;

chrome/browser/vr/ui_host/vr_ui_host_impl.cc

@@ -228,6 +228,7 @@ void VRUiHostImpl::SetWebXRWebContents(content::WebContents* contents) {
     StartUiRendering();
     InitCapturingStates();
     ui_rendering_thread_->SetWebXrPresenting(true);
+    ui_rendering_thread_->SetFramesThrottled(frames_throttled_);
 
     PollCapturingState();
 
@@ -257,6 +258,17 @@ void VRUiHostImpl::SetWebXRWebContents(content::WebContents* contents) {
   }
 }
 
+void VRUiHostImpl::SetFramesThrottled(bool throttled) {
+  frames_throttled_ = throttled;
+
+  if (!ui_rendering_thread_) {
+    DVLOG(1) << __func__ << ": no ui_rendering_thread_";
+    return;
+  }
+
+  ui_rendering_thread_->SetFramesThrottled(frames_throttled_);
+}
+
 void VRUiHostImpl::SetVRDisplayInfo(
     device::mojom::VRDisplayInfoPtr display_info) {
   DCHECK_CALLED_ON_VALID_THREAD(thread_checker_);

chrome/browser/vr/ui_host/vr_ui_host_impl.h

@@ -73,6 +73,7 @@ class VRUiHostImpl : public VRUiHost,
   // BrowserXRRuntimeObserver implementation.
   void SetWebXRWebContents(content::WebContents* contents) override;
   void SetVRDisplayInfo(device::mojom::VRDisplayInfoPtr display_info) override;
+  void SetFramesThrottled(bool throttled) override;
 
   // Internal methods used to start/stop the UI rendering thread that is used
   // for drawing browser UI (such as permission prompts) for display in VR.
@@ -120,6 +121,7 @@ class VRUiHostImpl : public VRUiHost,
   base::Time indicators_shown_start_time_;
   bool indicators_visible_ = false;
   bool indicators_showing_first_time_ = true;
+  bool frames_throttled_ = false;
 
   mojo::Remote<device::mojom::GeolocationConfig> geolocation_config_;
   base::CancelableClosure poll_capturing_state_task_;

chrome/browser/vr/win/vr_browser_renderer_thread_win.cc

@@ -95,12 +95,15 @@ void VRBrowserRendererThreadWin::SetWebXrPresenting(bool presenting) {
 }
 
 void VRBrowserRendererThreadWin::StartWebXrTimeout() {
-  waiting_for_first_frame_ = true;
+  frame_timeout_running_ = true;
   overlay_->SetOverlayAndWebXRVisibility(draw_state_.ShouldDrawUI(),
                                          draw_state_.ShouldDrawWebXR());
 
-  overlay_->RequestNotificationOnWebXrSubmitted(base::BindOnce(
-      &VRBrowserRendererThreadWin::OnWebXRSubmitted, base::Unretained(this)));
+  if (!waiting_for_webxr_frame_) {
+    waiting_for_webxr_frame_ = true;
+    overlay_->RequestNotificationOnWebXrSubmitted(base::BindOnce(
+        &VRBrowserRendererThreadWin::OnWebXRSubmitted, base::Unretained(this)));
+  }
 
   webxr_spinner_timeout_closure_.Reset(base::BindOnce(
       &VRBrowserRendererThreadWin::OnWebXrTimeoutImminent,
@@ -124,7 +127,7 @@ void VRBrowserRendererThreadWin::StopWebXrTimeout() {
   if (!webxr_frame_timeout_closure_.IsCancelled())
     webxr_frame_timeout_closure_.Cancel();
   OnSpinnerVisibilityChanged(false);
-  waiting_for_first_frame_ = false;
+  frame_timeout_running_ = false;
 }
 
 int VRBrowserRendererThreadWin::GetNextRequestId() {
@@ -144,16 +147,10 @@ void VRBrowserRendererThreadWin::OnWebXrTimedOut() {
   scheduler_ui_->OnWebXrTimedOut();
 }
 
-void VRBrowserRendererThreadWin::SetVisibleExternalPromptNotification(
-    ExternalPromptNotificationType prompt) {
-  if (!draw_state_.SetPrompt(prompt))
-    return;
-
+void VRBrowserRendererThreadWin::UpdateOverlayState() {
   if (draw_state_.ShouldDrawUI())
     StartOverlay();
 
-  ui_->SetVisibleExternalPromptNotification(prompt);
-
   if (overlay_)
     overlay_->SetOverlayAndWebXRVisibility(draw_state_.ShouldDrawUI(),
                                            draw_state_.ShouldDrawWebXR());
@@ -167,26 +164,55 @@ void VRBrowserRendererThreadWin::SetVisibleExternalPromptNotification(
   }
 }
 
-void VRBrowserRendererThreadWin::SetIndicatorsVisible(bool visible) {
-  if (!draw_state_.SetIndicatorsVisible(visible))
+void VRBrowserRendererThreadWin::SetFramesThrottled(bool throttled) {
+  if (frames_throttled_ == throttled)
     return;
 
-  if (draw_state_.ShouldDrawUI())
-    StartOverlay();
+  frames_throttled_ = throttled;
 
-  if (overlay_)
-    overlay_->SetOverlayAndWebXRVisibility(draw_state_.ShouldDrawUI(),
-                                           draw_state_.ShouldDrawWebXR());
-  if (draw_state_.ShouldDrawUI()) {
-    if (overlay_)  // False only while testing
-      overlay_->RequestNextOverlayPose(
-          base::BindOnce(&VRBrowserRendererThreadWin::OnPose,
-                         base::Unretained(this), GetNextRequestId()));
+  if (g_frame_timeout_ui_disabled_for_testing_)
+    return;
+
+  if (frames_throttled_) {
+    StopWebXrTimeout();
+
+    // TODO(alcooper): This is not necessarily the best thing to show, but it's
+    // the best that we have right now.  It ensures that we submit *something*
+    // rather than letting the default system "Stalled" UI take over, without
+    // showing a message that the page is behaving badly.
+    OnWebXrTimeoutImminent();
   } else {
-    StopOverlay();
+    StartWebXrTimeout();
   }
 }
 
+void VRBrowserRendererThreadWin::SetVisibleExternalPromptNotification(
+    ExternalPromptNotificationType prompt) {
+  if (!draw_state_.SetPrompt(prompt))
+    return;
+
+  UpdateOverlayState();
+
+  if (!ui_) {
+    // If the ui is dismissed, make sure that we don't *actually* have a prompt
+    // state that we needed to set.
+    DCHECK(prompt == ExternalPromptNotificationType::kPromptNone);
+    return;
+  }
+
+  ui_->SetVisibleExternalPromptNotification(prompt);
+}
+
+void VRBrowserRendererThreadWin::SetIndicatorsVisible(bool visible) {
+  if (draw_state_.SetIndicatorsVisible(visible))
+    UpdateOverlayState();
+}
+
+void VRBrowserRendererThreadWin::OnSpinnerVisibilityChanged(bool visible) {
+  if (draw_state_.SetSpinnerVisible(visible))
+    UpdateOverlayState();
+}
+
 void VRBrowserRendererThreadWin::SetCapturingState(
     const CapturingStateModel& active_capturing,
     const CapturingStateModel& background_capturing,
@@ -313,31 +339,11 @@ void VRBrowserRendererThreadWin::StartOverlay() {
   started_ = true;
 }
 
-void VRBrowserRendererThreadWin::OnSpinnerVisibilityChanged(bool visible) {
-  if (!draw_state_.SetSpinnerVisible(visible))
-    return;
-  if (draw_state_.ShouldDrawUI()) {
-    StartOverlay();
-  }
-
-  if (overlay_) {
-    overlay_->SetOverlayAndWebXRVisibility(draw_state_.ShouldDrawUI(),
-                                           draw_state_.ShouldDrawWebXR());
-  }
-
-  if (draw_state_.ShouldDrawUI()) {
-    if (overlay_)  // False only while testing.
-      overlay_->RequestNextOverlayPose(
-          base::BindOnce(&VRBrowserRendererThreadWin::OnPose,
-                         base::Unretained(this), GetNextRequestId()));
-  } else {
-    StopOverlay();
-  }
-}
-
 void VRBrowserRendererThreadWin::OnWebXRSubmitted() {
+  waiting_for_webxr_frame_ = false;
   if (scheduler_ui_)
     scheduler_ui_->OnWebXrFrameAvailable();
+
   StopWebXrTimeout();
 }
 
@@ -448,8 +454,12 @@ void VRBrowserRendererThreadWin::SubmitResult(bool success) {
   if (!success && graphics_) {
     graphics_->ResetMemoryBuffer();
   }
-  if (scheduler_ui_ && success && !waiting_for_first_frame_)
+
+  // Make sure that we only notify that a WebXr frame is now
+  if (scheduler_ui_ && success && !frame_timeout_running_) {
     scheduler_ui_->OnWebXrFrameAvailable();
+  }
+
   if (draw_state_.ShouldDrawUI() && started_) {
     overlay_->RequestNextOverlayPose(
         base::BindOnce(&VRBrowserRendererThreadWin::OnPose,

chrome/browser/vr/win/vr_browser_renderer_thread_win.h

@@ -34,6 +34,7 @@ class VR_EXPORT VRBrowserRendererThreadWin {
   void SetVRDisplayInfo(device::mojom::VRDisplayInfoPtr display_info);
   void SetLocationInfo(GURL gurl);
   void SetWebXrPresenting(bool presenting);
+  void SetFramesThrottled(bool throttled);
 
   // The below function(s) affect(s) whether UI is drawn or not.
   void SetVisibleExternalPromptNotification(
@@ -80,6 +81,8 @@ class VR_EXPORT VRBrowserRendererThreadWin {
   void StopWebXrTimeout();
   int GetNextRequestId();
 
+  void UpdateOverlayState();
+
   // We need to do some initialization of GraphicsDelegateWin before
   // browser_renderer_, so we first store it in a unique_ptr, then transition
   // ownership to browser_renderer_.
@@ -102,7 +105,9 @@ class VR_EXPORT VRBrowserRendererThreadWin {
   DrawState draw_state_;
   bool started_ = false;
   bool webxr_presenting_ = false;
-  bool waiting_for_first_frame_ = true;
+  bool frame_timeout_running_ = true;
+  bool waiting_for_webxr_frame_ = false;
+  bool frames_throttled_ = false;
   int current_request_id_ = 0;
 
   device::mojom::ImmersiveOverlayPtr overlay_;

device/vr/public/mojom/vr_service.mojom

@@ -455,6 +455,15 @@ interface VRService {
   GetImmersiveVRDisplayInfo() => (VRDisplayInfo? info);
 
   ExitPresent();
+
+  // Used for the renderer process to indicate that it is throttling frame
+  // requests from the page, and thus it (not the page) is responsible for any
+  // drop in frame rate or delay in posting frames. This is mainly meant to be
+  // informational for the browser process, so that it can decide if or what UI
+  // to show if it seems like the frame rate is too low or has stalled. The
+  // renderer can (and may) still decide to submit frames and that should not be
+  // treated as illegal or clear the throttled state.
+  SetFramesThrottled(bool throttled);
 };
 
 // The interface for the renderer to listen to top level XR events, events that

third_party/blink/renderer/modules/xr/xr.cc

@@ -530,6 +530,15 @@ void XR::ExitPresent() {
   }
 }
 
+void XR::SetFramesThrottled(const XRSession* session, bool throttled) {
+  // The service only cares if the immersive session is throttling frames.
+  if (session->immersive()) {
+    // If we have an immersive session, we should have a service.
+    DCHECK(service_);
+    service_->SetFramesThrottled(throttled);
+  }
+}
+
 ScriptPromise XR::supportsSession(ScriptState* script_state,
                                   const String& mode,
                                   ExceptionState& exception_state) {

third_party/blink/renderer/modules/xr/xr.h

@@ -92,6 +92,8 @@ class XR final : public EventTargetWithInlineData,
 
   void ExitPresent();
 
+  void SetFramesThrottled(const XRSession* session, bool throttled);
+
   base::TimeTicks NavigationStart() const { return navigation_start_; }
 
  private:

third_party/blink/renderer/modules/xr/xr_session.cc

@@ -909,13 +909,39 @@ void XRSession::MaybeRequestFrame() {
     }
   }
 
-  // We can request a frame if we're not hidden, we don't already have a pending
-  // frame, we have pending callbacks, and we will have a base layer when it
-  // resolves.
-  bool can_request_frame =
-      visibility_state_ != XRVisibilityState::HIDDEN && !pending_frame_ &&
-      !callback_collection_->IsEmpty() && will_have_base_layer;
-  if (can_request_frame) {
+  // A page will not be allowed to get frames if its visibility state is hidden.
+  bool page_allowed_frames = visibility_state_ != XRVisibilityState::HIDDEN;
+
+  // A page is configured properly if it will have a base layer when the frame
+  // callback gets resolved.
+  bool page_configured_properly = will_have_base_layer;
+
+  // If we have an outstanding callback registered, then we know that the page
+  // actually wants frames.
+  bool page_wants_frame = !callback_collection_->IsEmpty();
+
+  // A page can process frames if it has its appropriate base layer set and has
+  // indicated that it actually wants frames.
+  bool page_can_process_frames = page_configured_properly && page_wants_frame;
+
+  // We consider frames to be throttled if the page is not allowed frames, but
+  // otherwise would be able to receive them. Therefore, if the page isn't in a
+  // state to process frames, it doesn't matter if we are throttling it, any
+  // "stalls" should be attributed to the page being poorly behaved.
+  bool frames_throttled = page_can_process_frames && !page_allowed_frames;
+
+  // If our throttled state has changed, notify anyone who may care
+  if (frames_throttled_ != frames_throttled) {
+    frames_throttled_ = frames_throttled;
+    xr_->SetFramesThrottled(this, frames_throttled_);
+  }
+
+  // We can request a frame if we don't have one already pending, the page is
+  // allowed to request frames, and the page is set up to properly handle frames
+  // and wants one.
+  bool request_frame =
+      !pending_frame_ && page_allowed_frames && page_can_process_frames;
+  if (request_frame) {
     xr_->frameProvider()->RequestFrame(this);
     pending_frame_ = true;
   }

third_party/blink/renderer/modules/xr/xr_session.h

@@ -341,6 +341,7 @@ class XRSession final
   bool resolving_frame_ = false;
   bool update_views_next_frame_ = false;
   bool views_dirty_ = true;
+  bool frames_throttled_ = false;
 
   // Indicates that we've already logged a metric, so don't need to log it
   // again.

third_party/blink/web_tests/wpt_internal/webxr/resources/xr-internal-device-mocking.js

@@ -41,3 +41,21 @@ MockRuntime.prototype.getMissingFrameCount = function() {
 // Patch in experimental features.
 MockRuntime.featureToMojoMap["dom-overlay-for-handheld-ar"] =
     device.mojom.XRSessionFeature.DOM_OVERLAY_FOR_HANDHELD_AR;
+
+ChromeXRTest.prototype.getService = function() {
+  return this.mockVRService_;
+}
+
+MockVRService.prototype.setFramesThrottled = function(throttled) {
+  return this.frames_throttled_ = throttled;
+}
+
+MockVRService.prototype.getFramesThrottled = function() {
+  // Explicitly converted falsey states (i.e. undefined) to false.
+  if (!this.frames_throttled_) {
+    return false;
+  }
+
+  return this.frames_throttled_;
+};
+

third_party/blink/web_tests/wpt_internal/webxr/xrSession_framesThrottled.https.html

+<!DOCTYPE html>
+<script src="/resources/testharness.js"></script>
+<script src="/resources/testharnessreport.js"></script>
+<script>let additionalChromiumResources = ["resources/xr-internal-device-mocking.js"];</script>
+<script src="/webxr/resources/webxr_util.js"></script>
+<script src="/webxr/resources/webxr_test_constants.js"></script>
+<script src="/webxr/resources/webxr_test_asserts.js"></script>
+<canvas />
+
+<script>
+// While the logic of not handing out animation frames in certain scenarios is
+// specced, the notion of what the browser does during those cases is not.
+// This tests that blink appropriately reports throttled state to the browser
+// process. Note that the throttled state is defined as being in a state where
+// blink is not handing out poses to the page, but the page is properly set up.
+let testName =
+  "Blink appropriately reports when frames are throttled";
+
+let testFunction = function(session, fakeDeviceController, t) {
+
+  // We need to yield some reasonable amount of time so that mojo messages can
+  // propagate. This provides a convinience method to do this via promises and
+  // timeouts.
+  function runAfterTimeout(callback, timeout = 100) {
+    return new Promise((resolve) => {
+      t.step_timeout(() => {
+        resolve(callback());
+      }, timeout);
+    });
+  }
+
+  let service = navigator.xr.test.getService();
+  fakeDeviceController.simulateVisibilityChange("hidden");
+
+  function onXRFrame() {
+    session.requestAnimationFrame(onXRFrame);
+  }
+
+  return runAfterTimeout(() => {
+    assert_false(service.getFramesThrottled(),
+      "Without a frame loop, the page shouldn't be considered throttled");
+  })
+  .then(() => {
+    // Start a simple frame loop.
+    session.requestAnimationFrame(onXRFrame);
+
+    return runAfterTimeout(() => {
+      assert_true(service.getFramesThrottled(),
+        "With a properly configured frame loop, and a hidden device state, " +
+        "the page should be considered throttled.");
+    });
+  })
+  .then(() => {
+    fakeDeviceController.simulateVisibilityChange("visible");
+
+    return runAfterTimeout(() => {
+      assert_false(service.getFramesThrottled(),
+        "While properly configured and visible, frames should not be throttled");
+    });
+  });
+};
+
+xr_session_promise_test(
+  testName, testFunction, TRACKED_IMMERSIVE_DEVICE, 'immersive-vr');
+
+</script>