[telemetry] test_runner and run_benchmarks

- test.Test, which encapsulates a test and page set, and describes how to run it.
- test_runner, which runs test.Tests using a command-based CLI.

- benchmarks dir, which contains a list of test.Tests.
- run_measurement, which just calls test_runner.Main().

BUG=237412
TEST=None.
R=kbr@chromium.org, nduca@chromium.org, tonyg@chromium.org

Review URL: https://codereview.chromium.org/17438002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@207874 0039d316-1c4b-4281-b951-d872f2087c98

content/test/gpu/run_gpu_tests

@@ -40,9 +40,8 @@ if __name__ == '__main__':
                     'test/gpu/gpu_tests/bootstrap_deps')
   import gpu_tests
   from telemetry.page import page_test_runner
-  test_dir = os.path.join(os.path.dirname(__file__), 'gpu_tests')
 
   import page_sets # pylint: disable=F0401
   page_set_filenames = page_sets.GetAllPageSetFilenames()
 
-  sys.exit(page_test_runner.Main(test_dir, '', page_set_filenames))
+  sys.exit(page_test_runner.Main(os.path.dirname(__file__), page_set_filenames))

tools/perf/benchmarks/smoothness.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry import test
+
+from measurements import smoothness
+
+
+class SmoothnessTop25(test.Test):
+  """Measures rendering statistics while scrolling down the top 25 web pages"""
+  test = smoothness.Smoothness
+  page_set = 'page_sets/top_25.json'
+  options = {'report_all_results': False}

tools/perf/perf_tools/sunspider.py → tools/perf/benchmarks/sunspider.py

-# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
-
-"""Runs Apple's SunSpider JavaScript benchmark."""
-
 import collections
 import json
 import os
 
+from telemetry import test
 from telemetry.core import util
 from telemetry.page import page_measurement
 from telemetry.page import page_set
 
-class SunSpiderMeasurement(page_measurement.PageMeasurement):
-  def CreatePageSet(self, _, options):
-    return page_set.PageSet.FromDict({
-        'serving_dirs': ['../../../chrome/test/data/sunspider/'],
-        'pages': [
-          { 'url': 'file:///../../../chrome/test/data/sunspider/'
-                   'sunspider-1.0/driver.html' }
-          ]
-        }, os.path.abspath(__file__))
 
+class SunspiderMeasurement(page_measurement.PageMeasurement):
   def MeasurePage(self, _, tab, results):
-    js_is_done = """
-window.location.pathname.indexOf('results.html') >= 0"""
+    js_is_done = 'window.location.pathname.indexOf("results.html") >= 0'
     def _IsDone():
       return tab.EvaluateJavaScript(js_is_done)
-    util.WaitFor(_IsDone, 300, poll_interval=5)
+    util.WaitFor(_IsDone, 300, poll_interval=1)
 
     js_get_results = 'JSON.stringify(output);'
     js_results = json.loads(tab.EvaluateJavaScript(js_get_results))
@@ -45,3 +34,18 @@ window.location.pathname.indexOf('results.html') >= 0"""
     for key, values in r.iteritems():
       results.Add(key, 'ms', values, data_type='unimportant')
     results.Add('Total', 'ms', totals)
+
+
+class Sunspider(test.Test):
+  """Performance on Apple's SunSpider JavaScript benchmark"""
+  test = SunspiderMeasurement
+
+  def CreatePageSet(self, options):
+    sunspider_dir = os.path.join(util.GetChromiumSrcDir(),
+                                 'chrome', 'test', 'data', 'sunspider')
+    return page_set.PageSet.FromDict(
+        {
+          'serving_dirs': [''],
+          'pages': [{ 'url': 'file:///sunspider-1.0/driver.html' }],
+        },
+        sunspider_dir)

tools/perf/perf_tools/smoothness_measurement.py → tools/perf/measurements/smoothness.py

@@ -155,9 +155,9 @@ def CalcLatencyResults(rendering_stats_deltas, results):
               data_type='unimportant')
 
 
-class SmoothnessMeasurement(page_measurement.PageMeasurement):
+class Smoothness(page_measurement.PageMeasurement):
   def __init__(self):
-    super(SmoothnessMeasurement, self).__init__('smoothness')
+    super(Smoothness, self).__init__('smoothness')
     self.force_enable_threaded_compositing = False
     self.use_gpu_benchmarking_extension = True
     self._metrics = None

tools/perf/run_benchmark

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, 'telemetry'))
+
+from telemetry import test_runner
+
+if __name__ == '__main__':
+  sys.exit(test_runner.Main())

tools/perf/run_measurement

@@ -51,15 +51,12 @@ def main():
                     'http://src.chromium.org/viewvc/chrome/trunk/src/tools'
                     '/perf/perf_tools/' + DEPS_FILE)
   import perf_tools
-  import profile_creators
   if '--print-bootstrap-deps' in sys.argv:
     print ListBootstrapDeps()
     sys.exit(0)
 
   from telemetry.page import page_measurement_runner
   import page_sets
-  measurement_dir = os.path.dirname(perf_tools.__file__)
-  profile_creators_dir = os.path.dirname(profile_creators.__file__)
   page_set_filenames = page_sets.GetAllPageSetFilenames()
 
   old_benchmark_names = {
@@ -86,8 +83,7 @@ def main():
       return old_benchmark_names[arg]
 
   runner = MeasurementRunner()
-  sys.exit(
-      runner.Run(measurement_dir, profile_creators_dir, page_set_filenames))
+  sys.exit(runner.Run(os.path.dirname(__file__), page_set_filenames))
 
 if __name__ == '__main__':
   sys.exit(main())

tools/telemetry/docs/telemetry.core.chrome.android_browser_finder.html

@@ -86,7 +86,7 @@ Data descriptors inherited from <a href="telemetry.core.possible_browser.html#Po
 <font color="#ffffff" face="helvetica, arial"><big><strong>Functions</strong></big></font></td></tr>
     
 <tr><td bgcolor="#eeaa77"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
-<td width="100%"><dl><dt><a name="-FindAllAvailableBrowsers"><strong>FindAllAvailableBrowsers</strong></a>(options, logging<font color="#909090">=&lt;module 'logging' from '/System/Library/Framewor...Versions/2.7/lib/python2.7/logging/__init__.pyc'&gt;</font>)</dt><dd><tt>Finds&nbsp;all&nbsp;the&nbsp;desktop&nbsp;browsers&nbsp;available&nbsp;on&nbsp;this&nbsp;machine.</tt></dd></dl>
+<td width="100%"><dl><dt><a name="-FindAllAvailableBrowsers"><strong>FindAllAvailableBrowsers</strong></a>(options, logging<font color="#909090">=&lt;module 'logging' from '/usr/lib/python2.7/logging/__init__.pyc'&gt;</font>)</dt><dd><tt>Finds&nbsp;all&nbsp;the&nbsp;desktop&nbsp;browsers&nbsp;available&nbsp;on&nbsp;this&nbsp;machine.</tt></dd></dl>
 </td></tr></table><p>
 <table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">
 <tr bgcolor="#55aa55">

tools/telemetry/docs/telemetry.core.chrome.desktop_browser_backend.html

@@ -20,15 +20,13 @@
     
 <tr><td bgcolor="#aa55cc"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="telemetry.core.chrome.browser_backend.html">telemetry.core.chrome.browser_backend</a><br>
-<a href="telemetry.core.chrome.cros_interface.html">telemetry.core.chrome.cros_interface</a><br>
-<a href="telemetry.core.chrome.cros_util.html">telemetry.core.chrome.cros_util</a><br>
-</td><td width="25%" valign=top><a href="logging.html">logging</a><br>
-<a href="os.html">os</a><br>
+<a href="logging.html">logging</a><br>
+</td><td width="25%" valign=top><a href="os.html">os</a><br>
 <a href="shutil.html">shutil</a><br>
 </td><td width="25%" valign=top><a href="subprocess.html">subprocess</a><br>
 <a href="sys.html">sys</a><br>
-<a href="tempfile.html">tempfile</a><br>
-</td><td width="25%" valign=top><a href="telemetry.core.util.html">telemetry.core.util</a><br>
+</td><td width="25%" valign=top><a href="tempfile.html">tempfile</a><br>
+<a href="telemetry.core.util.html">telemetry.core.util</a><br>
 </td></tr></table></td></tr></table><p>
 <table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">
 <tr bgcolor="#ee77aa">
@@ -75,7 +73,7 @@ Methods defined here:<br>
 
 <dl><dt><a name="DesktopBrowserBackend-__del__"><strong>__del__</strong></a>(self)</dt></dl>
 
-<dl><dt><a name="DesktopBrowserBackend-__init__"><strong>__init__</strong></a>(self, options, executable, is_content_shell, use_login, delete_profile_dir_after_run<font color="#909090">=True</font>)</dt></dl>
+<dl><dt><a name="DesktopBrowserBackend-__init__"><strong>__init__</strong></a>(self, options, executable, is_content_shell, delete_profile_dir_after_run<font color="#909090">=True</font>)</dt></dl>
 
 <hr>
 Data descriptors defined here:<br>

tools/telemetry/docs/telemetry.core.chrome.desktop_browser_finder.html

@@ -63,7 +63,7 @@ Methods defined here:<br>
 
 <dl><dt><a name="PossibleDesktopBrowser-SupportsOptions"><strong>SupportsOptions</strong></a>(self, options)</dt></dl>
 
-<dl><dt><a name="PossibleDesktopBrowser-__init__"><strong>__init__</strong></a>(self, browser_type, options, executable, is_content_shell, use_login<font color="#909090">=False</font>)</dt></dl>
+<dl><dt><a name="PossibleDesktopBrowser-__init__"><strong>__init__</strong></a>(self, browser_type, options, executable, is_content_shell)</dt></dl>
 
 <dl><dt><a name="PossibleDesktopBrowser-__repr__"><strong>__repr__</strong></a>(self)</dt></dl>
 
@@ -94,5 +94,5 @@ Data descriptors inherited from <a href="telemetry.core.possible_browser.html#Po
 <font color="#ffffff" face="helvetica, arial"><big><strong>Data</strong></big></font></td></tr>
     
 <tr><td bgcolor="#55aa55"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
-<td width="100%"><strong>ALL_BROWSER_TYPES</strong> = 'exact,release,debug,canary,content-shell-debug,content-shell-release,debug-cros,release-cros,system'</td></tr></table>
+<td width="100%"><strong>ALL_BROWSER_TYPES</strong> = 'exact,release,debug,canary,content-shell-debug,content-shell-release,system'</td></tr></table>
 </body></html>
\ No newline at end of file

tools/telemetry/docs/telemetry.core.chrome.tracing_backend.html

@@ -22,9 +22,9 @@
 <td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="cStringIO.html">cStringIO</a><br>
 <a href="json.html">json</a><br>
 <a href="logging.html">logging</a><br>
-</td><td width="25%" valign=top><a href="socket.html">socket</a><br>
+</td><td width="25%" valign=top><a href="telemetry.core.timeline.model.html">telemetry.core.timeline.model</a><br>
+<a href="socket.html">socket</a><br>
 <a href="threading.html">threading</a><br>
-<a href="telemetry.core.timeline.trace_event_importer.html">telemetry.core.timeline.trace_event_importer</a><br>
 </td><td width="25%" valign=top><a href="telemetry.core.chrome.trace_result.html">telemetry.core.chrome.trace_result</a><br>
 <a href="telemetry.core.util.html">telemetry.core.util</a><br>
 <a href="telemetry.core.chrome.websocket.html">telemetry.core.chrome.websocket</a><br>

tools/telemetry/docs/telemetry.core.memory_cache_http_server.html

@@ -94,6 +94,9 @@ Static methods defined here:<br>
 
 <hr>
 Data descriptors defined here:<br>
+<dl><dt><strong>__dict__</strong></dt>
+<dd><tt>Return&nbsp;a&nbsp;new&nbsp;OrderedDict&nbsp;which&nbsp;maps&nbsp;field&nbsp;names&nbsp;to&nbsp;their&nbsp;values</tt></dd>
+</dl>
 <dl><dt><strong>from_byte</strong></dt>
 <dd><tt>Alias&nbsp;for&nbsp;field&nbsp;number&nbsp;0</tt></dd>
 </dl>
@@ -230,7 +233,7 @@ probably&nbsp;be&nbsp;diagnosed.)</tt></dd></dl>
 
 <hr>
 Data and other attributes inherited from <a href="SimpleHTTPServer.html#SimpleHTTPRequestHandler">SimpleHTTPServer.SimpleHTTPRequestHandler</a>:<br>
-<dl><dt><strong>extensions_map</strong> = {'': 'application/octet-stream', '.123': 'application/vnd.lotus-1-2-3', '.3dml': 'text/vnd.in3d.3dml', '.3g2': 'video/3gpp2', '.3gp': 'video/3gpp', '.7z': 'application/x-7z-compressed', '.a': 'application/octet-stream', '.aab': 'application/x-authorware-bin', '.aac': 'audio/x-aac', '.aam': 'application/x-authorware-map', ...}</dl>
+<dl><dt><strong>extensions_map</strong> = {'': 'application/octet-stream', '.%': 'application/x-trash', '.323': 'text/h323', '.3gp': 'video/3gpp', '.7z': 'application/x-7z-compressed', '.a': 'application/octet-stream', '.abw': 'application/x-abiword', '.ai': 'application/postscript', '.aif': 'audio/x-aiff', '.aifc': 'audio/x-aiff', ...}</dl>
 
 <dl><dt><strong>server_version</strong> = 'SimpleHTTP/0.6'</dl>
 
@@ -326,7 +329,7 @@ Data and other attributes inherited from <a href="BaseHTTPServer.html#BaseHTTPRe
 
 <dl><dt><strong>responses</strong> = {100: ('Continue', 'Request received, please continue'), 101: ('Switching Protocols', 'Switching to new protocol; obey Upgrade header'), 200: ('OK', 'Request fulfilled, document follows'), 201: ('Created', 'Document created, URL follows'), 202: ('Accepted', 'Request accepted, processing continues off-line'), 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), 204: ('No Content', 'Request fulfilled, nothing follows'), 205: ('Reset Content', 'Clear input form for further input.'), 206: ('Partial Content', 'Partial content follows.'), 300: ('Multiple Choices', 'Object has several resources -- see URI list'), ...}</dl>
 
-<dl><dt><strong>sys_version</strong> = 'Python/2.7.2'</dl>
+<dl><dt><strong>sys_version</strong> = 'Python/2.7.3'</dl>
 
 <dl><dt><strong>weekdayname</strong> = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun']</dl>
 
@@ -494,6 +497,9 @@ Static methods defined here:<br>
 
 <hr>
 Data descriptors defined here:<br>
+<dl><dt><strong>__dict__</strong></dt>
+<dd><tt>Return&nbsp;a&nbsp;new&nbsp;OrderedDict&nbsp;which&nbsp;maps&nbsp;field&nbsp;names&nbsp;to&nbsp;their&nbsp;values</tt></dd>
+</dl>
 <dl><dt><strong>byte_range</strong></dt>
 <dd><tt>Alias&nbsp;for&nbsp;field&nbsp;number&nbsp;1</tt></dd>
 </dl>

tools/telemetry/docs/telemetry.core.timeline.event.html

@@ -39,7 +39,14 @@
 <td width="100%">Methods defined here:<br>
 <dl><dt><a name="TimelineEvent-GetAllChildrenRecursive"><strong>GetAllChildrenRecursive</strong></a>(self, include_self<font color="#909090">=False</font>)</dt></dl>
 
-<dl><dt><a name="TimelineEvent-__init__"><strong>__init__</strong></a>(self, name, start, duration, args<font color="#909090">=None</font>)</dt></dl>
+<dl><dt><a name="TimelineEvent-ShiftTimestampsForward"><strong>ShiftTimestampsForward</strong></a>(self, delta_time)</dt><dd><tt>Shifts&nbsp;start&nbsp;time&nbsp;of&nbsp;event&nbsp;by&nbsp;delta_time&nbsp;and&nbsp;also<br>
+recursively&nbsp;shifts&nbsp;child&nbsp;events.</tt></dd></dl>
+
+<dl><dt><a name="TimelineEvent-UpdateBounds"><strong>UpdateBounds</strong></a>(self)</dt><dd><tt>Updates&nbsp;the&nbsp;start&nbsp;time&nbsp;to&nbsp;be&nbsp;the&nbsp;minimum&nbsp;start&nbsp;time&nbsp;of&nbsp;all<br>
+child&nbsp;events&nbsp;and&nbsp;the&nbsp;end&nbsp;time&nbsp;to&nbsp;be&nbsp;the&nbsp;maximum&nbsp;end&nbsp;time&nbsp;of&nbsp;all<br>
+child&nbsp;events.</tt></dd></dl>
+
+<dl><dt><a name="TimelineEvent-__init__"><strong>__init__</strong></a>(self, name, start, duration, args<font color="#909090">=None</font>, parent<font color="#909090">=None</font>)</dt></dl>
 
 <dl><dt><a name="TimelineEvent-__repr__"><strong>__repr__</strong></a>(self)</dt></dl>
 

tools/telemetry/docs/telemetry.core.timeline.html

@@ -19,11 +19,12 @@
 <tr><td bgcolor="#aa55cc"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="telemetry.core.timeline.event.html">event</a><br>
 <a href="telemetry.core.timeline.event_unittest.html">event_unittest</a><br>
-</td><td width="25%" valign=top><a href="telemetry.core.timeline.importer.html">importer</a><br>
-<a href="telemetry.core.timeline.inspector_importer.html">inspector_importer</a><br>
-</td><td width="25%" valign=top><a href="telemetry.core.timeline.inspector_importer_unittest.html">inspector_importer_unittest</a><br>
+<a href="telemetry.core.timeline.importer.html">importer</a><br>
+</td><td width="25%" valign=top><a href="telemetry.core.timeline.inspector_importer.html">inspector_importer</a><br>
+<a href="telemetry.core.timeline.inspector_importer_unittest.html">inspector_importer_unittest</a><br>
 <a href="telemetry.core.timeline.model.html">model</a><br>
 </td><td width="25%" valign=top><a href="telemetry.core.timeline.model_unittest.html">model_unittest</a><br>
 <a href="telemetry.core.timeline.trace_event_importer.html">trace_event_importer</a><br>
-</td></tr></table></td></tr></table>
+<a href="telemetry.core.timeline.tracing.html"><strong>tracing</strong>&nbsp;(package)</a><br>
+</td><td width="25%" valign=top></td></tr></table></td></tr></table>
 </body></html>
\ No newline at end of file

tools/telemetry/docs/telemetry.core.timeline.model.html

@@ -21,7 +21,8 @@ https://code.google.com/p/trace-viewer/</tt></p>
     
 <tr><td bgcolor="#aa55cc"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="telemetry.core.timeline.inspector_importer.html">telemetry.core.timeline.inspector_importer</a><br>
-</td><td width="25%" valign=top></td><td width="25%" valign=top></td><td width="25%" valign=top></td></tr></table></td></tr></table><p>
+</td><td width="25%" valign=top><a href="telemetry.core.timeline.tracing.trace_event_importer.html">telemetry.core.timeline.tracing.trace_event_importer</a><br>
+</td><td width="25%" valign=top></td><td width="25%" valign=top></td></tr></table></td></tr></table><p>
 <table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">
 <tr bgcolor="#ee77aa">
 <td colspan=3 valign=bottom>&nbsp;<br>
@@ -50,11 +51,17 @@ https://code.google.com/p/trace-viewer/</tt></p>
 
 <dl><dt><a name="TimelineModel-GetAllEvents"><strong>GetAllEvents</strong></a>(self)</dt></dl>
 
-<dl><dt><a name="TimelineModel-GetAllOfName"><strong>GetAllOfName</strong></a>(self, name)</dt></dl>
+<dl><dt><a name="TimelineModel-GetAllEventsOfName"><strong>GetAllEventsOfName</strong></a>(self, name)</dt></dl>
 
-<dl><dt><a name="TimelineModel-ImportTraces"><strong>ImportTraces</strong></a>(self, traces)</dt></dl>
+<dl><dt><a name="TimelineModel-GetRootEvents"><strong>GetRootEvents</strong></a>(self)</dt></dl>
 
-<dl><dt><a name="TimelineModel-__init__"><strong>__init__</strong></a>(self)</dt></dl>
+<dl><dt><a name="TimelineModel-ImportTraces"><strong>ImportTraces</strong></a>(self, traces, shift_world_to_zero<font color="#909090">=True</font>)</dt></dl>
+
+<dl><dt><a name="TimelineModel-ShiftWorldToZero"><strong>ShiftWorldToZero</strong></a>(self)</dt></dl>
+
+<dl><dt><a name="TimelineModel-UpdateBounds"><strong>UpdateBounds</strong></a>(self)</dt></dl>
+
+<dl><dt><a name="TimelineModel-__init__"><strong>__init__</strong></a>(self, event_data<font color="#909090">=None</font>, shift_world_to_zero<font color="#909090">=True</font>)</dt></dl>
 
 <hr>
 Data descriptors defined here:<br>
@@ -64,5 +71,9 @@ Data descriptors defined here:<br>
 <dl><dt><strong>__weakref__</strong></dt>
 <dd><tt>list&nbsp;of&nbsp;weak&nbsp;references&nbsp;to&nbsp;the&nbsp;object&nbsp;(if&nbsp;defined)</tt></dd>
 </dl>
+<dl><dt><strong>max_timestamp</strong></dt>
+</dl>
+<dl><dt><strong>min_timestamp</strong></dt>
+</dl>
 </td></tr></table></td></tr></table>
 </body></html>
\ No newline at end of file

tools/telemetry/docs/telemetry.core.timeline.trace_event_importer.html

-
-<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
-<html><head><title>Python: module telemetry.core.timeline.trace_event_importer</title>
-</head><body bgcolor="#f0f0f8">
-
-<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="heading">
-<tr bgcolor="#7799ee">
-<td valign=bottom>&nbsp;<br>
-<font color="#ffffff" face="helvetica, arial">&nbsp;<br><big><big><strong><a href="telemetry.html"><font color="#ffffff">telemetry</font></a>.<a href="telemetry.core.html"><font color="#ffffff">core</font></a>.<a href="telemetry.core.timeline.html"><font color="#ffffff">timeline</font></a>.trace_event_importer</strong></big></big></font></td
-><td align=right valign=bottom
-><font color="#ffffff" face="helvetica, arial"><a href=".">index</a><br><a href="../telemetry/core/timeline/trace_event_importer.py">telemetry/core/timeline/trace_event_importer.py</a></font></td></tr></table>
-    <p><tt>#&nbsp;Copyright&nbsp;(c)&nbsp;2012&nbsp;The&nbsp;Chromium&nbsp;Authors.&nbsp;All&nbsp;rights&nbsp;reserved.<br>
-#&nbsp;Use&nbsp;of&nbsp;this&nbsp;source&nbsp;code&nbsp;is&nbsp;governed&nbsp;by&nbsp;a&nbsp;BSD-style&nbsp;license&nbsp;that&nbsp;can&nbsp;be<br>
-#&nbsp;found&nbsp;in&nbsp;the&nbsp;LICENSE&nbsp;file.</tt></p>
-<p>
-<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">
-<tr bgcolor="#aa55cc">
-<td colspan=3 valign=bottom>&nbsp;<br>
-<font color="#ffffff" face="helvetica, arial"><big><strong>Modules</strong></big></font></td></tr>
-    
-<tr><td bgcolor="#aa55cc"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
-<td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="json.html">json</a><br>
-</td><td width="25%" valign=top><a href="telemetry.core.timeline.model.html">telemetry.core.timeline.model</a><br>
-</td><td width="25%" valign=top></td><td width="25%" valign=top></td></tr></table></td></tr></table><p>
-<table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">
-<tr bgcolor="#eeaa77">
-<td colspan=3 valign=bottom>&nbsp;<br>
-<font color="#ffffff" face="helvetica, arial"><big><strong>Functions</strong></big></font></td></tr>
-    
-<tr><td bgcolor="#eeaa77"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
-<td width="100%"><dl><dt><a name="-Import"><strong>Import</strong></a>(data)</dt></dl>
-</td></tr></table>
-</body></html>
\ No newline at end of file

tools/telemetry/docs/telemetry.html

@@ -18,7 +18,9 @@
     
 <tr><td bgcolor="#aa55cc"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="telemetry.core.html"><strong>core</strong>&nbsp;(package)</a><br>
-</td><td width="25%" valign=top><a href="telemetry.page.html"><strong>page</strong>&nbsp;(package)</a><br>
+<a href="telemetry.page.html"><strong>page</strong>&nbsp;(package)</a><br>
+</td><td width="25%" valign=top><a href="telemetry.test.html">test</a><br>
+<a href="telemetry.test_runner.html">test_runner</a><br>
 </td><td width="25%" valign=top><a href="telemetry.unittest.html"><strong>unittest</strong>&nbsp;(package)</a><br>
 </td><td width="25%" valign=top></td></tr></table></td></tr></table><p>
 <table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">

tools/telemetry/docs/telemetry.page.csv_page_measurement_results.html

@@ -56,7 +56,7 @@ Methods defined here:<br>
 
 <dl><dt><a name="CsvPageMeasurementResults-PrintSummary"><strong>PrintSummary</strong></a>(self)</dt></dl>
 
-<dl><dt><a name="CsvPageMeasurementResults-__init__"><strong>__init__</strong></a>(self, output_stream, output_after_every_page, trace_tag<font color="#909090">=''</font>)</dt></dl>
+<dl><dt><a name="CsvPageMeasurementResults-__init__"><strong>__init__</strong></a>(self, output_stream, output_after_every_page)</dt></dl>
 
 <hr>
 Methods inherited from <a href="telemetry.page.page_measurement_results.html#PageMeasurementResults">telemetry.page.page_measurement_results.PageMeasurementResults</a>:<br>

tools/telemetry/docs/telemetry.page.html

@@ -26,25 +26,24 @@
 <a href="telemetry.page.buildbot_page_measurement_results_unittest.html">buildbot_page_measurement_results_unittest</a><br>
 <a href="telemetry.page.csv_page_measurement_results.html">csv_page_measurement_results</a><br>
 <a href="telemetry.page.csv_page_measurement_results_unittest.html">csv_page_measurement_results_unittest</a><br>
-<a href="telemetry.page.failure.html">failure</a><br>
-</td><td width="25%" valign=top><a href="telemetry.page.gtest_test_results.html">gtest_test_results</a><br>
-<a href="telemetry.page.page.html">page</a><br>
+<a href="telemetry.page.gtest_test_results.html">gtest_test_results</a><br>
+</td><td width="25%" valign=top><a href="telemetry.page.page.html">page</a><br>
 <a href="telemetry.page.page_filter.html">page_filter</a><br>
 <a href="telemetry.page.page_measurement.html">page_measurement</a><br>
 <a href="telemetry.page.page_measurement_results.html">page_measurement_results</a><br>
 <a href="telemetry.page.page_measurement_results_unittest.html">page_measurement_results_unittest</a><br>
 <a href="telemetry.page.page_measurement_runner.html">page_measurement_runner</a><br>
 <a href="telemetry.page.page_measurement_unittest.html">page_measurement_unittest</a><br>
-</td><td width="25%" valign=top><a href="telemetry.page.page_measurement_unittest_base.html">page_measurement_unittest_base</a><br>
-<a href="telemetry.page.page_measurement_value.html">page_measurement_value</a><br>
+<a href="telemetry.page.page_measurement_unittest_base.html">page_measurement_unittest_base</a><br>
+</td><td width="25%" valign=top><a href="telemetry.page.page_measurement_value.html">page_measurement_value</a><br>
 <a href="telemetry.page.page_runner.html">page_runner</a><br>
 <a href="telemetry.page.page_runner_unittest.html">page_runner_unittest</a><br>
 <a href="telemetry.page.page_set.html">page_set</a><br>
 <a href="telemetry.page.page_set_archive_info.html">page_set_archive_info</a><br>
 <a href="telemetry.page.page_set_archive_info_unittest.html">page_set_archive_info_unittest</a><br>
 <a href="telemetry.page.page_set_unittest.html">page_set_unittest</a><br>
-</td><td width="25%" valign=top><a href="telemetry.page.page_test.html">page_test</a><br>
-<a href="telemetry.page.page_test_results.html">page_test_results</a><br>
+<a href="telemetry.page.page_test.html">page_test</a><br>
+</td><td width="25%" valign=top><a href="telemetry.page.page_test_results.html">page_test_results</a><br>
 <a href="telemetry.page.page_test_runner.html">page_test_runner</a><br>
 <a href="telemetry.page.page_test_unittest.html">page_test_unittest</a><br>
 <a href="telemetry.page.page_unittest.html">page_unittest</a><br>

tools/telemetry/docs/telemetry.page.page_measurement_runner.html

@@ -57,7 +57,7 @@ Data descriptors defined here:<br>
 </dl>
 <hr>
 Methods inherited from <a href="telemetry.page.page_test_runner.html#PageTestRunner">telemetry.page.page_test_runner.PageTestRunner</a>:<br>
-<dl><dt><a name="PageMeasurementRunner-FindTestConstructors"><strong>FindTestConstructors</strong></a>(self, test_dir)</dt></dl>
+<dl><dt><a name="PageMeasurementRunner-FindTestConstructors"><strong>FindTestConstructors</strong></a>(self, base_dir)</dt></dl>
 
 <dl><dt><a name="PageMeasurementRunner-FindTestName"><strong>FindTestName</strong></a>(self, test_constructors, args)</dt><dd><tt>Find&nbsp;the&nbsp;test&nbsp;name&nbsp;in&nbsp;an&nbsp;arbitrary&nbsp;argument&nbsp;list.<br>
 &nbsp;<br>
@@ -75,11 +75,11 @@ Otherwise,&nbsp;return&nbsp;the&nbsp;arg.</tt></dd></dl>
 
 <dl><dt><a name="PageMeasurementRunner-GetPageSet"><strong>GetPageSet</strong></a>(self, test, page_set_filenames)</dt></dl>
 
-<dl><dt><a name="PageMeasurementRunner-ParseCommandLine"><strong>ParseCommandLine</strong></a>(self, args, test_dir, profile_creators_dir, page_set_filenames)</dt></dl>
+<dl><dt><a name="PageMeasurementRunner-ParseCommandLine"><strong>ParseCommandLine</strong></a>(self, args, base_dir, page_set_filenames)</dt></dl>
 
 <dl><dt><a name="PageMeasurementRunner-PrintParseError"><strong>PrintParseError</strong></a>(self, message)</dt></dl>
 
-<dl><dt><a name="PageMeasurementRunner-Run"><strong>Run</strong></a>(self, test_dir, profile_creators_dir, page_set_filenames)</dt></dl>
+<dl><dt><a name="PageMeasurementRunner-Run"><strong>Run</strong></a>(self, base_dir, page_set_filenames)</dt></dl>
 
 <dl><dt><a name="PageMeasurementRunner-__init__"><strong>__init__</strong></a>(self)</dt></dl>
 
@@ -98,10 +98,9 @@ Data descriptors inherited from <a href="telemetry.page.page_test_runner.html#Pa
 <font color="#ffffff" face="helvetica, arial"><big><strong>Functions</strong></big></font></td></tr>
     
 <tr><td bgcolor="#eeaa77"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
-<td width="100%"><dl><dt><a name="-Main"><strong>Main</strong></a>(measurement_dir, profile_creators_dir, page_set_filenames)</dt><dd><tt>Turns&nbsp;a&nbsp;PageMeasurement&nbsp;into&nbsp;a&nbsp;command-line&nbsp;program.<br>
+<td width="100%"><dl><dt><a name="-Main"><strong>Main</strong></a>(base_dir, page_set_filenames)</dt><dd><tt>Turns&nbsp;a&nbsp;PageMeasurement&nbsp;into&nbsp;a&nbsp;command-line&nbsp;program.<br>
 &nbsp;<br>
 Args:<br>
-&nbsp;&nbsp;measurement_dir:&nbsp;Path&nbsp;to&nbsp;directory&nbsp;containing&nbsp;PageMeasurements.<br>
-&nbsp;&nbsp;profile_creators_dir:&nbsp;Path&nbsp;to&nbsp;directory&nbsp;containing&nbsp;ProfileCreators.</tt></dd></dl>
+&nbsp;&nbsp;base_dir:&nbsp;Path&nbsp;to&nbsp;directory&nbsp;containing&nbsp;tests&nbsp;and&nbsp;ProfileCreators.</tt></dd></dl>
 </td></tr></table>
 </body></html>
\ No newline at end of file

tools/telemetry/docs/telemetry.page.page_runner.html

@@ -59,7 +59,7 @@
 <td width="100%">Methods defined here:<br>
 <dl><dt><a name="PageState-CleanUpPage"><strong>CleanUpPage</strong></a>(self, page, tab)</dt></dl>
 
-<dl><dt><a name="PageState-PreparePage"><strong>PreparePage</strong></a>(self, page, tab, test)</dt></dl>
+<dl><dt><a name="PageState-PreparePage"><strong>PreparePage</strong></a>(self, page, tab, test<font color="#909090">=None</font>)</dt></dl>
 
 <dl><dt><a name="PageState-__init__"><strong>__init__</strong></a>(self)</dt></dl>
 

tools/telemetry/docs/telemetry.page.page_test_runner.html

@@ -21,13 +21,14 @@
 <tr><td bgcolor="#aa55cc"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%"><table width="100%" summary="list"><tr><td width="25%" valign=top><a href="telemetry.core.browser_options.html">telemetry.core.browser_options</a><br>
 <a href="telemetry.core.discover.html">telemetry.core.discover</a><br>
-</td><td width="25%" valign=top><a href="os.html">os</a><br>
-<a href="telemetry.page.page_runner.html">telemetry.page.page_runner</a><br>
-</td><td width="25%" valign=top><a href="telemetry.page.page_set.html">telemetry.page.page_set</a><br>
+<a href="os.html">os</a><br>
+</td><td width="25%" valign=top><a href="telemetry.page.page_runner.html">telemetry.page.page_runner</a><br>
+<a href="telemetry.page.page_set.html">telemetry.page.page_set</a><br>
 <a href="telemetry.page.page_test.html">telemetry.page.page_test</a><br>
 </td><td width="25%" valign=top><a href="telemetry.core.profile_types.html">telemetry.core.profile_types</a><br>
 <a href="sys.html">sys</a><br>
-</td></tr></table></td></tr></table><p>
+<a href="telemetry.test.html">telemetry.test</a><br>
+</td><td width="25%" valign=top></td></tr></table></td></tr></table><p>
 <table width="100%" cellspacing=0 cellpadding=2 border=0 summary="section">
 <tr bgcolor="#ee77aa">
 <td colspan=3 valign=bottom>&nbsp;<br>
@@ -50,7 +51,7 @@
     
 <tr><td bgcolor="#ffc8d8"><tt>&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
 <td width="100%">Methods defined here:<br>
-<dl><dt><a name="PageTestRunner-FindTestConstructors"><strong>FindTestConstructors</strong></a>(self, test_dir)</dt></dl>
+<dl><dt><a name="PageTestRunner-FindTestConstructors"><strong>FindTestConstructors</strong></a>(self, base_dir)</dt></dl>
 
 <dl><dt><a name="PageTestRunner-FindTestName"><strong>FindTestName</strong></a>(self, test_constructors, args)</dt><dd><tt>Find&nbsp;the&nbsp;test&nbsp;name&nbsp;in&nbsp;an&nbsp;arbitrary&nbsp;argument&nbsp;list.<br>
 &nbsp;<br>
@@ -68,11 +69,11 @@ Otherwise,&nbsp;return&nbsp;the&nbsp;arg.</tt></dd></dl>
 
 <dl><dt><a name="PageTestRunner-GetPageSet"><strong>GetPageSet</strong></a>(self, test, page_set_filenames)</dt></dl>
 
-<dl><dt><a name="PageTestRunner-ParseCommandLine"><strong>ParseCommandLine</strong></a>(self, args, test_dir, profile_creators_dir, page_set_filenames)</dt></dl>
+<dl><dt><a name="PageTestRunner-ParseCommandLine"><strong>ParseCommandLine</strong></a>(self, args, base_dir, page_set_filenames)</dt></dl>
 
 <dl><dt><a name="PageTestRunner-PrintParseError"><strong>PrintParseError</strong></a>(self, message)</dt></dl>
 
-<dl><dt><a name="PageTestRunner-Run"><strong>Run</strong></a>(self, test_dir, profile_creators_dir, page_set_filenames)</dt></dl>
+<dl><dt><a name="PageTestRunner-Run"><strong>Run</strong></a>(self, base_dir, page_set_filenames)</dt></dl>
 
 <dl><dt><a name="PageTestRunner-__init__"><strong>__init__</strong></a>(self)</dt></dl>
 
@@ -95,10 +96,9 @@ Data descriptors defined here:<br>
 <font color="#ffffff" face="helvetica, arial"><big><strong>Functions</strong></big></font></td></tr>
     
 <tr><td bgcolor="#eeaa77"><tt>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;</tt></td><td>&nbsp;</td>
-<td width="100%"><dl><dt><a name="-Main"><strong>Main</strong></a>(test_dir, profile_creators_dir, page_set_filenames)</dt><dd><tt>Turns&nbsp;a&nbsp;PageTest&nbsp;into&nbsp;a&nbsp;command-line&nbsp;program.<br>
+<td width="100%"><dl><dt><a name="-Main"><strong>Main</strong></a>(base_dir, page_set_filenames)</dt><dd><tt>Turns&nbsp;a&nbsp;PageTest&nbsp;into&nbsp;a&nbsp;command-line&nbsp;program.<br>
 &nbsp;<br>
 Args:<br>
-&nbsp;&nbsp;test_dir:&nbsp;Path&nbsp;to&nbsp;directory&nbsp;containing&nbsp;PageTests.<br>
-&nbsp;&nbsp;profile_creators_dir:&nbsp;Path&nbsp;to&nbsp;a&nbsp;directory&nbsp;containing&nbsp;ProfileCreators</tt></dd></dl>
+&nbsp;&nbsp;base_dir:&nbsp;Path&nbsp;to&nbsp;directory&nbsp;containing&nbsp;tests&nbsp;and&nbsp;ProfileCreators.</tt></dd></dl>
 </td></tr></table>
 </body></html>
\ No newline at end of file

tools/telemetry/telemetry/core/profile_types.py

@@ -24,8 +24,7 @@ def _DiscoverCreateableProfiles(profile_creators_dir):
   'small_profile_creator will be 'small_profile'.
   """
   profile_creators_unfiltered = (
-      discover.DiscoverClasses(profile_creators_dir,
-          os.path.join(profile_creators_dir, '..'),
+      discover.DiscoverClasses(profile_creators_dir, profile_creators_dir,
           profile_creator.ProfileCreator))
 
   # Remove '_creator' suffix from keys.

tools/telemetry/telemetry/page/csv_page_measurement_results.py

@@ -7,8 +7,8 @@ from telemetry.page import page_measurement_results
 
 class CsvPageMeasurementResults(
     page_measurement_results.PageMeasurementResults):
-  def __init__(self, output_stream, output_after_every_page, trace_tag=''):
-    super(CsvPageMeasurementResults, self).__init__(trace_tag)
+  def __init__(self, output_stream, output_after_every_page):
+    super(CsvPageMeasurementResults, self).__init__()
     self._results_writer = csv.writer(output_stream)
     self._did_output_header = False
     self._header_names_written_to_writer = None

tools/telemetry/telemetry/page/page_measurement.py

@@ -70,16 +70,19 @@ class PageMeasurement(page_test.PageTest):
     return ['buildbot', 'block', 'csv', 'none']
 
   def PrepareResults(self, options):
-    if not options.output_file or options.output_file == '-':
-      output_stream = sys.stdout
-    else:
+    if hasattr(options, 'output_file') and options.output_file:
       output_stream = open(os.path.expanduser(options.output_file), 'w')
+    else:
+      output_stream = sys.stdout
+    if not hasattr(options, 'output_format'):
+      options.output_format = self.output_format_choices[0]
+    if not hasattr(options, 'output_trace_tag'):
+      options.output_trace_tag = ''
 
     if options.output_format == 'csv':
       return csv_page_measurement_results.CsvPageMeasurementResults(
         output_stream,
-        self.results_are_the_same_on_every_page,
-        trace_tag=options.output_trace_tag)
+        self.results_are_the_same_on_every_page)
     elif options.output_format == 'block':
       return block_page_measurement_results.BlockPageMeasurementResults(
         output_stream)

tools/telemetry/telemetry/page/page_measurement_runner.py

@@ -7,16 +7,14 @@ import sys
 from telemetry.page import page_measurement
 from telemetry.page import page_test_runner
 
-def Main(measurement_dir, profile_creators_dir, page_set_filenames):
+def Main(base_dir, page_set_filenames):
   """Turns a PageMeasurement into a command-line program.
 
   Args:
-    measurement_dir: Path to directory containing PageMeasurements.
-    profile_creators_dir: Path to directory containing ProfileCreators.
+    base_dir: Path to directory containing tests and ProfileCreators.
   """
   runner = PageMeasurementRunner()
-  sys.exit(runner.Run(
-      measurement_dir, profile_creators_dir, page_set_filenames))
+  sys.exit(runner.Run(base_dir, page_set_filenames))
 
 class PageMeasurementRunner(page_test_runner.PageTestRunner):
   @property

tools/telemetry/telemetry/page/page_test.py

@@ -147,6 +147,9 @@ class PageTest(object):
     return ['gtest', 'none']
 
   def PrepareResults(self, options):
+    if not hasattr(options, 'output_format'):
+      options.output_format = self.output_format_choices[0]
+
     if options.output_format == 'gtest':
       return gtest_test_results.GTestTestResults()
     elif options.output_format == 'none':

tools/telemetry/telemetry/page/page_test_runner.py

@@ -4,6 +4,7 @@
 import os
 import sys
 
+from telemetry import test as test_module
 from telemetry.core import browser_options
 from telemetry.core import discover
 from telemetry.core import profile_types
@@ -11,15 +12,14 @@ from telemetry.page import page_test
 from telemetry.page import page_runner
 from telemetry.page import page_set
 
-def Main(test_dir, profile_creators_dir, page_set_filenames):
+def Main(base_dir, page_set_filenames):
   """Turns a PageTest into a command-line program.
 
   Args:
-    test_dir: Path to directory containing PageTests.
-    profile_creators_dir: Path to a directory containing ProfileCreators
+    base_dir: Path to directory containing tests and ProfileCreators.
   """
   runner = PageTestRunner()
-  sys.exit(runner.Run(test_dir, profile_creators_dir, page_set_filenames))
+  sys.exit(runner.Run(base_dir, page_set_filenames))
 
 class PageTestRunner(object):
   def __init__(self):
@@ -35,16 +35,19 @@ class PageTestRunner(object):
   def test_class_name(self):
     return 'test'
 
-  def Run(self, test_dir, profile_creators_dir, page_set_filenames):
-    test, ps = self.ParseCommandLine(
-        sys.argv, test_dir, profile_creators_dir, page_set_filenames)
+  def Run(self, base_dir, page_set_filenames):
+    test, ps = self.ParseCommandLine(sys.argv, base_dir, page_set_filenames)
     results = page_runner.Run(test, ps, self._options)
     results.PrintSummary()
     return min(255, len(results.failures + results.errors))
 
-  def FindTestConstructors(self, test_dir):
-    return discover.DiscoverClasses(
-        test_dir, os.path.join(test_dir, '..'), self.test_class)
+  def FindTestConstructors(self, base_dir):
+    # Look for both Tests and PageTests, but Tests get priority, because
+    test_constructors = discover.DiscoverClasses(
+        base_dir, base_dir, self.test_class)
+    test_constructors.update(discover.DiscoverClasses(
+        base_dir, base_dir, test_module.Test, index_by_class_name=True))
+    return test_constructors
 
   def FindTestName(self, test_constructors, args):
     """Find the test name in an arbitrary argument list.
@@ -122,23 +125,22 @@ class PageTestRunner(object):
     raise Exception('Did not understand "%s". Pass a page set, file or URL.' %
                     page_set_arg)
 
-  def ParseCommandLine(self, args, test_dir, profile_creators_dir,
-      page_set_filenames):
+  def ParseCommandLine(self, args, base_dir, page_set_filenames):
     # Need to collect profile creators before creating command line parser.
-    if profile_creators_dir:
-      profile_types.FindProfileCreators(profile_creators_dir)
+    profile_types.FindProfileCreators(base_dir)
 
     self._options = browser_options.BrowserOptions()
     self._parser = self._options.CreateParser(
         '%%prog [options] %s page_set' % self.test_class_name)
 
-    test_constructors = self.FindTestConstructors(test_dir)
+    test_constructors = self.FindTestConstructors(base_dir)
     test_name = self.FindTestName(test_constructors, args)
     test = None
     if test_name:
       test = test_constructors[test_name]()
-      test.AddOutputOptions(self._parser)
-      test.AddCommandLineOptions(self._parser)
+      if not isinstance(test, test_module.Test):
+        test.AddOutputOptions(self._parser)
+        test.AddCommandLineOptions(self._parser)
     page_runner.AddCommandLineOptions(self._parser)
 
     _, self._args = self._parser.parse_args()
@@ -155,12 +157,18 @@ class PageTestRunner(object):
       test_list_string = ',\n'.join(sorted(test_constructors.keys()))
       self.PrintParseError(error_message + test_list_string)
 
-    ps = self.GetPageSet(test, page_set_filenames)
+    if isinstance(test, test_module.Test):
+      ps = test.CreatePageSet(self._options)
+    else:
+      ps = self.GetPageSet(test, page_set_filenames)
 
     if len(self._args) > 2:
       self.PrintParseError('Too many arguments.')
 
-    return test, ps
+    if isinstance(test, test_module.Test):
+      return test.test(), ps
+    else:
+      return test, ps
 
   def PrintParseError(self, message):
     self._parser.error(message)

tools/telemetry/telemetry/test.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry.page import page_runner
+from telemetry.page import page_set
+from telemetry.page import page_test
+
+
+class Test(object):
+  """Base class for a Telemetry test or benchmark.
+
+  A test packages a PageTest/PageMeasurement and a PageSet together.
+  """
+  options = {}
+  enabled = True
+
+  def Run(self, options):
+    """Run this test with the given options."""
+    assert hasattr(self, 'test'), 'This test has no "test" attribute.'
+    assert issubclass(self.test, page_test.PageTest), (
+            '"%s" is not a PageTest.' % self.test.__name__)
+
+    for key, value in self.options.iteritems():
+      setattr(options, key, value)
+
+    test = self.test()
+    ps = self.CreatePageSet(options)
+    results = page_runner.Run(test, ps, options)
+    results.PrintSummary()
+    return len(results.failures) + len(results.errors)
+
+  def CreatePageSet(self, options):  # pylint: disable=W0613
+    """Get the page set this test will run on.
+
+    By default, it will create a page set from the file at this test's
+    page_set attribute. Override to generate a custom page set.
+    """
+    assert hasattr(self, 'page_set'), 'This test has no "page_set" attribute.'
+    return page_set.PageSet.FromFile(self.page_set)

tools/telemetry/telemetry/test_runner.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Parses the command line, discovers the appropriate tests, and runs them.
+
+Handles test configuration, but all the logic for
+actually running the test is in Test and PageRunner."""
+
+import copy
+import inspect
+import json
+import optparse
+import os
+import sys
+
+from telemetry import test
+from telemetry.core import browser_options
+from telemetry.core import discover
+
+
+class Command(object):
+  usage = ''
+
+  @property
+  def name(self):
+    return self.__class__.__name__.lower()
+
+  @property
+  def description(self):
+    return self.__doc__
+
+  def CreateParser(self):
+    return optparse.OptionParser('%%prog %s %s' % (self.name, self.usage))
+
+  def AddParserOptions(self, parser):
+    pass
+
+  def ValidateCommandLine(self, parser, options, args):
+    pass
+
+  def Run(self, options, args):
+    raise NotImplementedError()
+
+
+class Help(Command):
+  """Display help information"""
+
+  def Run(self, options, args):
+    print >> sys.stderr, ('usage: %s <command> [<args>]' % _GetScriptName())
+    print >> sys.stderr, 'Available commands are:'
+    for command in COMMANDS:
+      print >> sys.stderr, '  %-10s %s' % (command.name, command.description)
+    return 0
+
+
+class List(Command):
+  """Lists the available tests"""
+
+  def AddParserOptions(self, parser):
+    parser.add_option('-j', '--json', action='store_true')
+
+  def Run(self, options, args):
+    if options.json:
+      test_list = []
+      for test_name, test_class in sorted(_GetTests().items()):
+        test_list.append({
+              'name': test_name,
+              'description': test_class.__doc__,
+              'enabled': test_class.enabled,
+              'options': test_class.options,
+            })
+      print >> sys.stderr, json.dumps(test_list)
+    else:
+      print >> sys.stderr, 'Available tests are:'
+      for test_name, test_class in sorted(_GetTests().items()):
+        print >> sys.stderr, '  %-20s %s' % (test_name,
+            test_class.__doc__.splitlines()[0])
+    return 0
+
+
+class Run(Command):
+  """Run one or more tests"""
+
+  usage = 'test_name [...] [<args>]'
+
+  def CreateParser(self):
+    options = browser_options.BrowserOptions()
+    parser = options.CreateParser('%%prog %s %s' % (self.name, self.usage))
+    return parser
+
+  def ValidateCommandLine(self, parser, options, args):
+    if not args:
+      parser.error('Must provide at least one test name')
+    for test_name in args:
+      if test_name not in _GetTests():
+        parser.error('No test named "%s"' % test_name)
+
+  def Run(self, options, args):
+    total_failures = 0
+    for test_name in args:
+      test_failures = _GetTests()[test_name]().Run(copy.copy(options))
+      total_failures += test_failures
+
+    return min(255, total_failures)
+
+
+COMMANDS = [cls() for _, cls in inspect.getmembers(sys.modules[__name__])
+            if inspect.isclass(cls)
+            and cls is not Command and issubclass(cls, Command)]
+
+
+def _GetScriptName():
+  return os.path.basename(sys.argv[0])
+
+
+def _GetBaseDir():
+  main_module = sys.modules['__main__']
+  if hasattr(main_module, '__file__'):
+    return os.path.dirname(main_module.__file__)
+  else:
+    return os.getcwd()
+
+
+def _GetTests():
+  # Lazy load and cache results.
+  if not hasattr(_GetTests, 'tests'):
+    base_dir = _GetBaseDir()
+    _GetTests.tests = discover.DiscoverClasses(base_dir, base_dir, test.Test,
+                                               index_by_class_name=True)
+  return _GetTests.tests
+
+
+def Main():
+  # Get the command name from the command line.
+  if len(sys.argv) > 1 and sys.argv[1] == '--help':
+    sys.argv[1] = 'help'
+
+  command_name = 'run'
+  for arg in sys.argv[1:]:
+    if not arg.startswith('-'):
+      command_name = arg
+      break
+
+  # Validate and interpret the command name.
+  commands = [command for command in COMMANDS
+              if command.name.startswith(command_name)]
+  if len(commands) > 1:
+    print >> sys.stderr, ('"%s" is not a %s command. Did you mean one of these?'
+                          % (command_name, _GetScriptName()))
+    for command in commands:
+      print >> sys.stderr, '  %-10s %s' % (command.name, command.description)
+    return 1
+  if commands:
+    command = commands[0]
+  else:
+    command = Run()
+
+  # Parse and run the command.
+  parser = command.CreateParser()
+  command.AddParserOptions(parser)
+  options, args = parser.parse_args()
+  if commands:
+    args = args[1:]
+  command.ValidateCommandLine(parser, options, args)
+  return command.Run(options, args)