telemetry: handle tests that are not supported on a platform.

When a test or measurement was not supported on a platform, we used to print
a warning and call sys.exit(0). This caused problems with unittests, because
it was considered a failure (for page measurements it was not a failure).

This patch introduces the page_test.TestNotSupportedOnPlatformFailure
exception. Tests and measurements throw this exception if they can't run on
a platform. PageTestRunner and unit tests catch these exceptions and
print a warning, but they don't consider it an error.

R=tonyg@chromium.org, dtu@chromium.org, vmpstr@chromium.org
BUG=

Review URL: https://codereview.chromium.org/100443007

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@260475 0039d316-1c4b-4281-b951-d872f2087c98

tools/perf/measurements/rasterize_and_record.py

@@ -2,11 +2,11 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import logging
 import time
 
 from metrics import rendering_stats
 from telemetry.page import page_measurement
+from telemetry.page import page_test
 from telemetry.page.perf_tests_helper import FlattenList
 import telemetry.core.timeline.bounds as timeline_bounds
 from telemetry.core.timeline.model import TimelineModel
@@ -62,9 +62,9 @@ class RasterizeAndRecord(page_measurement.PageMeasurement):
 
   def MeasurePage(self, page, tab, results):
     if not self._compositing_features_enabled:
-      logging.warning('Warning: compositing feature status unknown or not '+
-                      'forced and threaded. Skipping measurement.')
-      return
+      raise page_test.TestNotSupportedOnPlatformFailure(
+          'Compositing feature status unknown or not '+
+          'forced and threaded. Skipping measurement.')
 
     # Rasterize only what's visible.
     tab.ExecuteJavaScript(

tools/perf/measurements/rasterize_and_record_micro.py

@@ -2,12 +2,12 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import logging
 import sys
 import time
 
 from telemetry.core.util import TimeoutException
 from telemetry.page import page_measurement
+from telemetry.page import page_test
 
 class RasterizeAndRecordMicro(page_measurement.PageMeasurement):
   def __init__(self):
@@ -50,7 +50,9 @@ class RasterizeAndRecordMicro(page_measurement.PageMeasurement):
     backend = browser._browser_backend # pylint: disable=W0212
     if (not hasattr(backend, 'chrome_branch_number') or
         (sys.platform != 'android' and backend.chrome_branch_number < 1713)):
-      return
+      raise page_test.TestNotSupportedOnPlatformFailure(
+          'rasterize_and_record_micro requires Chrome branch 1713 '
+          'or later. Skipping measurement.')
 
     # Check if the we actually have threaded forced compositing enabled.
     system_info = browser.GetSystemInfo()
@@ -61,9 +63,9 @@ class RasterizeAndRecordMicro(page_measurement.PageMeasurement):
 
   def MeasurePage(self, page, tab, results):
     if not self._compositing_features_enabled:
-      logging.warning('Warning: RasterizeAndRecordMicro requires forced, '
-                      'threaded compositing and Chrome branch 1713 or newer.')
-      return
+      raise page_test.TestNotSupportedOnPlatformFailure(
+          'Compositing feature status unknown or not '+
+          'forced and threaded. Skipping measurement.')
 
     try:
       tab.WaitForJavaScriptExpression("document.readyState == 'complete'", 10)
@@ -131,4 +133,3 @@ class RasterizeAndRecordMicro(page_measurement.PageMeasurement):
           total_picture_layers_with_no_content)
       results.Add('total_picture_layers_off_screen', 'count',
           total_picture_layers_off_screen)
-

tools/perf/measurements/rasterize_and_record_micro_unittest.py

 # Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+
+import logging
+
 from measurements import rasterize_and_record_micro
 from telemetry import test
 from telemetry.core import wpr_modes
 from telemetry.page import page_measurement_unittest_base
+from telemetry.page import page_test
 from telemetry.unittest import options_for_unittests
 
 
@@ -29,7 +33,11 @@ class RasterizeAndRecordMicroUnitTest(
   def testRasterizeAndRecordMicro(self):
     ps = self.CreatePageSetFromFileInUnittestDataDir('blank.html')
     measurement = rasterize_and_record_micro.RasterizeAndRecordMicro()
-    results = self.RunMeasurement(measurement, ps, options=self._options)
+    try:
+      results = self.RunMeasurement(measurement, ps, options=self._options)
+    except page_test.TestNotSupportedOnPlatformFailure as failure:
+      logging.warning(str(failure))
+      return
     self.assertEquals(0, len(results.failures))
 
     rasterize_time = results.FindAllPageSpecificValuesNamed('rasterize_time')

tools/perf/measurements/rasterize_and_record_unittest.py

@@ -2,10 +2,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import logging
+
 from measurements import rasterize_and_record
 from telemetry import test
 from telemetry.core import wpr_modes
 from telemetry.page import page_measurement_unittest_base
+from telemetry.page import page_test
 from telemetry.unittest import options_for_unittests
 
 
@@ -29,7 +32,11 @@ class RasterizeAndRecordUnitTest(
   def testRasterizeAndRecord(self):
     ps = self.CreatePageSetFromFileInUnittestDataDir('blank.html')
     measurement = rasterize_and_record.RasterizeAndRecord()
-    results = self.RunMeasurement(measurement, ps, options=self._options)
+    try:
+      results = self.RunMeasurement(measurement, ps, options=self._options)
+    except page_test.TestNotSupportedOnPlatformFailure as failure:
+      logging.warning(str(failure))
+      return
     self.assertEquals(0, len(results.failures))
 
     rasterize_time = results.FindAllPageSpecificValuesNamed('rasterize_time')

tools/telemetry/telemetry/page/page_measurement_unittest_base.py

@@ -8,6 +8,7 @@ from telemetry.core import util
 from telemetry.page import page_runner
 from telemetry.page import page as page_module
 from telemetry.page import page_set
+from telemetry.page import page_test
 from telemetry.page import test_expectations
 from telemetry.unittest import options_for_unittests
 
@@ -80,6 +81,9 @@ class PageMeasurementUnitTestBase(unittest.TestCase):
         return measurement_class.TabForPage(self, page, browser)
 
     measurement = BuggyMeasurement()
-    self.RunMeasurement(measurement, ps, options=options)
+    try:
+      self.RunMeasurement(measurement, ps, options=options)
+    except page_test.TestNotSupportedOnPlatformFailure:
+      pass
     if start_tracing_called[0]:
       self.assertTrue(stop_tracing_called[0])

tools/telemetry/telemetry/page/page_runner.py

@@ -478,6 +478,8 @@ def _RunPage(test, page, state, expectation, results, finder_options):
       page_state.ImplicitPageNavigation(test)
     test.RunPage(page, page_state.tab, results)
     util.CloseConnections(page_state.tab)
+  except page_test.TestNotSupportedOnPlatformFailure:
+    raise
   except page_test.Failure:
     if expectation == 'fail':
       logging.info('%s:\n%s', page.url, traceback.format_exc())

tools/telemetry/telemetry/page/page_test.py

@@ -62,6 +62,13 @@ class Failure(Exception):
   pass
 
 
+class TestNotSupportedOnPlatformFailure(Failure):
+  """Exception that can be thrown to indicate that a certain feature required
+  to run the test is not available on the platform, hardware configuration, or
+  browser version."""
+  pass
+
+
 class PageTest(command_line.Command):
   """A class styled on unittest.TestCase for creating page-specific tests."""
 

tools/telemetry/telemetry/page/page_test_runner.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+import logging
 import os
 import sys
 
 from telemetry import test as test_module
 from telemetry.core import browser_options
 from telemetry.core import discover
-from telemetry.page import page_test as page_test_module
 from telemetry.page import page_runner
 from telemetry.page import page_set
+from telemetry.page import page_test as page_test_module
+from telemetry.page import page_test_results
 
 def Main(env):
   """Turns a PageTest into a command-line program.
@@ -36,7 +38,11 @@ class PageTestRunner(object):
 
   def Run(self, env):
     test, ps, expectations = self.ParseCommandLine(sys.argv, env)
-    results = page_runner.Run(test, ps, expectations, self._options)
+    results = page_test_results.PageTestResults()
+    try:
+      results = page_runner.Run(test, ps, expectations, self._options)
+    except page_test_module.TestNotSupportedOnPlatformFailure as failure:
+      logging.warning(str(failure))
     results.PrintSummary()
     return min(255, len(results.failures + results.errors))
 

tools/telemetry/telemetry/test.py

@@ -17,6 +17,7 @@ from telemetry.page import page_runner
 from telemetry.page import cloud_storage
 from telemetry.page import page_set
 from telemetry.page import page_test
+from telemetry.page import page_test_results
 from telemetry.page import test_expectations
 
 
@@ -78,7 +79,12 @@ class Test(command_line.Command):
 
     self._DownloadGeneratedProfileArchive(args)
 
-    results = page_runner.Run(test, ps, expectations, args)
+    results = page_test_results.PageTestResults()
+    try:
+      results = page_runner.Run(test, ps, expectations, args)
+    except page_test.TestNotSupportedOnPlatformFailure as failure:
+      logging.warning(str(failure))
+
     results.PrintSummary()
     return len(results.failures) + len(results.errors)