Support VR perf tests on Windows

Adds support for running the existing VR Telemetry benchmarks on
Windows. Currently only supports the Oculus runtime, which requires
special hardware and software to be installed, but additional runtime
support will be added in follow-up CLs.

Bug: 939178
Change-Id: I71056b1da9aa07a374602757f5ddfafcfc972316
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1521797Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#641697}

BUILD.gn

@@ -757,15 +757,13 @@ group("gn_all") {
       "//chrome/browser/vr:vr_common_perftests",
       "//chrome/browser/vr:vr_common_unittests",
       "//chrome/browser/vr:vr_pixeltests",
+      "//tools/perf/contrib/vr_benchmarks:vr_perf_tests",
     ]
     if (is_desktop_linux && use_ozone) {
       deps += [ "//chrome/browser/vr/testapp:vr_testapp" ]
     }
     if (is_android) {
-      deps += [
-        "//chrome/browser/android/vr:vr_android_unittests",
-        "//tools/perf/contrib/vr_benchmarks:vr_perf_tests",
-      ]
+      deps += [ "//chrome/browser/android/vr:vr_android_unittests" ]
     }
   }
 

tools/perf/contrib/vr_benchmarks/BUILD.gn

@@ -4,14 +4,13 @@
 
 import("//chrome/browser/vr/features.gni")
 
-assert(is_android)
-
 group("vr_perf_tests") {
   testonly = true
   data = [
     "./data/",
+    "./desktop_runtimes/",
     "./__init__.py",
-    "./shared_android_vr_page_state.py",
+    "./shared_vr_page_state.py",
     "./vr_benchmarks.py",
     "./vr_browsing_mode_pages.py",
     "./vr_sample_page.py",
@@ -19,26 +18,33 @@ group("vr_perf_tests") {
     "./webvr_sample_pages.py",
     "./webvr_wpr_pages.py",
     "./webxr_sample_pages.py",
-    "//chrome/android/shared_preference_files/test/",
-    "//third_party/gvr-android-sdk/test-apks/vr_services/vr_services_current.apk",
-    "//third_party/gvr-android-sdk/test-apks/vr_keyboard/vr_keyboard_current.apk",
     "//chrome/test/data/xr/webvr_info/samples/",
     "//third_party/webxr_test_pages/webxr-samples",
   ]
   data_deps = [
-    "//chrome/android:vr_nfc_simulator_apk",
     "//testing:run_perf_test",
   ]
 
-  # We'll only ever use the assets if it's a Chrome-branded build. We don't have
-  # a way of checking whether the files are actually present to copy, but the
-  # script will deal with that.
-  if (use_vr_assets_component) {
-    data_deps += [ ":generate_vr_assets_profile" ]
-  }
   deps = [
     "//tools/perf:perf",
   ]
+
+  if (is_android) {
+    data += [
+      "//chrome/android/shared_preference_files/test/",
+      "//third_party/gvr-android-sdk/test-apks/vr_services/vr_services_current.apk",
+      "//third_party/gvr-android-sdk/test-apks/vr_keyboard/vr_keyboard_current.apk",
+    ]
+
+    data_deps += [ "//chrome/android:vr_nfc_simulator_apk" ]
+
+    # We'll only ever use the assets if it's a Chrome-branded build. We don't have
+    # a way of checking whether the files are actually present to copy, but the
+    # script will deal with that.
+    if (use_vr_assets_component) {
+      data_deps += [ ":generate_vr_assets_profile" ]
+    }
+  }
 }
 
 # Copies files to the gen/ directory and creates a manifest so that the VR

tools/perf/contrib/vr_benchmarks/desktop_runtimes/base_runtime.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+class DesktopRuntimeBase(object):
+  """Interface for all desktop VR runtimes."""
+
+  def __init__(self, finder_options):
+    self._finder_options = finder_options
+
+  def Setup(self):
+    """Called once before any stories are run."""
+    self._finder_options.browser_options.AppendExtraBrowserArgs(
+        '--enable-features=%s' % self.GetFeatureName())
+    self._SetupInternal()
+
+  def _SetupInternal(self):
+    raise NotImplementedError(
+        'No runtime setup defined for %s' % self.__class__.__name__)
+
+  def WillRunStory(self):
+    """Called before each story is run."""
+    self._WillRunStoryInternal()
+
+  def _WillRunStoryInternal(self):
+    raise NotImplementedError(
+        'No runtime pre-story defined for %s' % self.__class__.__name__)
+
+  def TearDown(self):
+    """Called once after all stories are run."""
+    self._TearDownInternal()
+
+  def _TearDownInternal(self):
+    raise NotImplementedError(
+        'No runtime tear down defined for %s' % self.__class__.__name__)
+
+  def GetFeatureName(self):
+    raise NotImplementedError(
+        'No feature defined for %s' % self.__class__.__name__)

tools/perf/contrib/vr_benchmarks/desktop_runtimes/oculus_runtimes.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import subprocess
+from contrib.vr_benchmarks.desktop_runtimes import base_runtime
+
+
+# pylint: disable=abstract-method
+class _BaseOculusRuntime(base_runtime.DesktopRuntimeBase):
+  """Base class for all Oculus runtimes."""
+
+  def __init__(self, *args, **kwargs):
+    super(_BaseOculusRuntime, self).__init__(*args, **kwargs)
+
+  def GetFeatureName(self):
+    return 'OculusVR'
+
+
+class OculusRuntimeReal(_BaseOculusRuntime):
+  """Class for using the real Oculus runtime for desktop tests."""
+
+  OCULUS_BASE_ENVIRONMENT_VARIABLE = 'OculusBase'
+
+  def __init__(self, *args, **kwargs):
+    super(OculusRuntimeReal, self).__init__(*args, **kwargs)
+    self._runtime_handle = None
+
+  def _SetupInternal(self):
+    # We need to launch the Oculus client before running any tests to ensure
+    # that the runtime is ready when we try to enter VR.
+    self._runtime_handle = subprocess.Popen([self._GetOculusClientPath()])
+
+  def _WillRunStoryInternal(self):
+    if not self._runtime_handle:
+      raise RuntimeError(
+          'Somehow called real Oculus pre-story without calling setup')
+    if self._runtime_handle.poll() != None:
+      logging.warning(
+          'Oculus client closed prematurely with code %d, restarting',
+          self._runtime_handle.returncode)
+      self._runtime_handle = subprocess.Popen([self._GetOculusClientPath()])
+
+  def _TearDownInternal(self):
+    if not self._runtime_handle:
+      raise RuntimeError(
+          'Somehow called real Oculus tear down without calling setup')
+    if self._runtime_handle.poll() is None:
+      self._runtime_handle.terminate()
+
+  def _GetOculusClientPath(self):
+    # The install location of the Oculus runtime is set in the OculusBase
+    # environment variable at install time.
+    if self.OCULUS_BASE_ENVIRONMENT_VARIABLE not in os.environ:
+      raise RuntimeError('Failed to find the Oculus install location. Are you '
+                         'sure it\'s installed?')
+    return os.path.join(os.environ[self.OCULUS_BASE_ENVIRONMENT_VARIABLE],
+                        'Support', 'oculus-client', 'OculusClient.exe')
+
+
+class OculusRuntimeMock(base_runtime.DesktopRuntimeBase):
+  """Class for using a mock Oculus runtime for desktop tests."""

tools/perf/contrib/vr_benchmarks/desktop_runtimes/openvr_runtimes.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from contrib.vr_benchmarks.desktop_runtimes import base_runtime
+
+
+# pylint: disable=abstract-method
+class _OpenVRRuntimeBase(base_runtime.DesktopRuntimeBase):
+  """Base class for all OpenVR runtimes."""
+
+  def GetFeatureName(self):
+    return 'OpenVR'
+
+
+class OpenVRRuntimeReal(_OpenVRRuntimeBase):
+  """Class for using the real OpenVR runtime for desktop tests."""
+
+
+class OpenVRRuntimeMock(_OpenVRRuntimeBase):
+  """Class for using the mock OpenVR runtime for desktop tests."""

tools/perf/contrib/vr_benchmarks/desktop_runtimes/wmr_runtimes.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from contrib.vr_benchmarks.desktop_runtimes import base_runtime
+
+
+# pylint: disable=abstract-method
+class _WMRRuntimeBase(base_runtime.DesktopRuntimeBase):
+  """Base class for all WMR runtimes."""
+
+  def GetFeatureName(self):
+    return 'WindowsMixedReality'
+
+
+class WMRRuntimeReal(_WMRRuntimeBase):
+  """Class for using the real Windows Mixed Reality runtime for desktop tests.
+  """
+
+
+class WMRRuntimeMock(_WMRRuntimeBase):
+  """Class for using the mock Windows Mixed Reality runtime for desktop tests.
+  """

tools/perf/contrib/vr_benchmarks/vr_benchmarks.py

@@ -11,6 +11,7 @@ from telemetry import story
 from telemetry.timeline import chrome_trace_category_filter
 from telemetry.timeline import chrome_trace_config
 from telemetry.web_perf import timeline_based_measurement
+from contrib.vr_benchmarks import shared_vr_page_state as vr_state
 from contrib.vr_benchmarks import vr_browsing_mode_pages
 from contrib.vr_benchmarks import webvr_sample_pages
 from contrib.vr_benchmarks import webvr_wpr_pages
@@ -19,6 +20,16 @@ from contrib.vr_benchmarks import webxr_sample_pages
 
 class _BaseVRBenchmark(perf_benchmark.PerfBenchmark):
 
+  # Trace categories that should be enabled for all VR benchmarks.
+  COMMON_TRACE_CATEGORIES = [
+      '-*', # Remove all default categories.
+      'blink.console', # Necessary for memory measurements.
+      'disabled-by-default-memory-infra', # Necessary for memory measurements.
+      'gpu', # Necessary for various VR metrics.
+      'toplevel', # Debug category.
+      'viz', # Debug category.
+    ]
+
   @classmethod
   def AddBenchmarkCommandLineArgs(cls, parser):
     parser.add_option(
@@ -59,24 +70,37 @@ class _BaseVRBenchmark(perf_benchmark.PerfBenchmark):
              'for it. This largely boils down to adding waits/sleeps in order '
              'to ensure that enough streaming data is recorded for the '
              'benchmark to run without issues.')
+    parser.add_option(
+        '--desktop-runtime',
+        default='openvr',
+        choices=vr_state.WindowsSharedVrPageState.DESKTOP_RUNTIMES.keys(),
+        help='Which VR runtime to use on Windows. Defaults to %default')
+    parser.add_option(
+        '--use-real-runtime',
+        action='store_true',
+        default=False,
+        help='Use the real runtime instead of a mock implementation. This '
+             'requires the runtime to be installed on the system.')
 
 
 class _BaseWebVRWebXRBenchmark(_BaseVRBenchmark):
 
-  SUPPORTED_PLATFORMS = [story.expectations.ALL_ANDROID]
+  SUPPORTED_PLATFORMS = [
+      story.expectations.ALL_ANDROID,
+      story.expectations.WIN_10
+  ]
 
   def CreateCoreTimelineBasedMeasurementOptions(self):
-    memory_categories = ['blink.console', 'disabled-by-default-memory-infra']
-    gpu_categories = ['gpu']
-    debug_categories = ['toplevel', 'viz']
-    category_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
-        ','.join(['-*'] + memory_categories + gpu_categories
-            + debug_categories))
+    category_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter()
+    for category in self.COMMON_TRACE_CATEGORIES:
+      category_filter.AddFilter(category)
+
     options = timeline_based_measurement.Options(category_filter)
     options.config.enable_android_graphics_memtrack = True
     options.config.enable_platform_display_trace = True
 
-    options.SetTimelineBasedMetrics(['memoryMetric', 'webvrMetric'])
+    options.SetTimelineBasedMetrics(
+        ['memoryMetric', 'webvrMetric', 'webxrMetric'])
     options.config.chrome_trace_config.SetMemoryDumpConfig(
         chrome_trace_config.MemoryDumpConfig())
     return options
@@ -172,12 +196,10 @@ class _BaseBrowsingBenchmark(_BaseVRBenchmark):
   SUPPORTED_PLATFORMS = [story.expectations.ALL_ANDROID]
 
   def CreateTimelineBasedMeasurementOptions(self):
-    memory_categories = ['blink.console', 'disabled-by-default-memory-infra']
-    gpu_categories = ['gpu']
-    debug_categories = ['toplevel', 'viz']
-    category_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
-        ','.join(['-*'] + memory_categories + gpu_categories
-            + debug_categories))
+    category_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter()
+    for category in self.COMMON_TRACE_CATEGORIES:
+      category_filter.AddFilter(category)
+
     options = timeline_based_measurement.Options(category_filter)
     options.config.enable_android_graphics_memtrack = True
     options.config.enable_platform_display_trace = True

tools/perf/contrib/vr_benchmarks/vr_browsing_mode_pages.py

@@ -8,7 +8,7 @@ from telemetry import page
 from telemetry import story
 from telemetry.page import shared_page_state
 from devil.android.sdk import intent  # pylint: disable=import-error
-from contrib.vr_benchmarks import shared_android_vr_page_state as vr_state
+from contrib.vr_benchmarks import shared_vr_page_state as vr_state
 from contrib.vr_benchmarks.vr_sample_page import VrSamplePage
 from contrib.vr_benchmarks.vr_story_set import VrStorySet
 from page_sets import top_10_mobile
@@ -72,7 +72,7 @@ class VrBrowsingModeWprPage(page.Page):
         page_set=page_set,
         name=name,
         extra_browser_args=extra_browser_args,
-        shared_page_state_class=vr_state.SharedAndroidVrPageState)
+        shared_page_state_class=vr_state.AndroidSharedVrPageState)
     self._shared_page_state = None
 
   def RunPageInteractions(self, action_runner):

tools/perf/contrib/vr_benchmarks/vr_sample_page.py

@@ -5,8 +5,7 @@
 import os
 import re
 from telemetry import page
-from contrib.vr_benchmarks import (shared_android_vr_page_state as
-                                   vr_state)
+from contrib.vr_benchmarks import shared_vr_page_state as vr_state
 
 WEBVR_SAMPLE_DIR = os.path.join(
     os.path.dirname(__file__), '..', '..', '..', '..', 'chrome', 'test',
@@ -40,7 +39,7 @@ class _VrXrSamplePage(page.Page):
         page_set=page_set,
         name=name,
         extra_browser_args=extra_browser_args,
-        shared_page_state_class=vr_state.SharedAndroidVrPageState)
+        shared_page_state_class=vr_state.SharedVrPageStateFactory)
     self._shared_page_state = None
 
   def Run(self, shared_state):

tools/perf/contrib/vr_benchmarks/webvr_wpr_pages.py

@@ -4,7 +4,7 @@
 
 from telemetry import story
 from telemetry import page
-from contrib.vr_benchmarks import (shared_android_vr_page_state as vr_state)
+from contrib.vr_benchmarks import (shared_vr_page_state as vr_state)
 from contrib.vr_benchmarks.vr_story_set import VrStorySet
 
 class WebVrWprPage(page.Page):
@@ -27,7 +27,7 @@ class WebVrWprPage(page.Page):
         page_set=page_set,
         name=name,
         extra_browser_args=extra_browser_args,
-        shared_page_state_class=vr_state.SharedAndroidVrPageState)
+        shared_page_state_class=vr_state.SharedVrPageStateFactory)
     self._shared_page_state = None
     self._interaction_function = interaction_function