Add a Telemetry based cold startup test.

This shares most of its code with the startup_warm measurement. You
must now specify either --warm or --cold to run the new startup
measurement.

Due to splitting up the original patch for this into many pieces, some
of the earlier changes didn't all work together. There are also a
couple of fixes for those in here.

BUG=None

Review URL: https://chromiumcodereview.appspot.com/22300013

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@217126 0039d316-1c4b-4281-b951-d872f2087c98

tools/perf/benchmarks/startup_warm.py → tools/perf/benchmarks/startup.py

-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 from telemetry import test
 
-from measurements import startup_warm
+from measurements import startup
+
+
+class StartupColdBlankPage(test.Test):
+  test = startup.Startup
+  page_set = 'page_sets/blank_page.json'
+  options = {'cold': True,
+             'pageset_repeat_iters': 5}
 
 
 class StartupWarmBlankPage(test.Test):
-  test = startup_warm.StartupWarm
+  test = startup.Startup
   page_set = 'page_sets/blank_page.json'
+  options = {'warm': True,
+             'pageset_repeat_iters': 20}

tools/perf/measurements/startup_warm.py → tools/perf/measurements/startup.py

-# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Copyright 2013 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -6,9 +6,15 @@ import json
 
 from telemetry.page import page_measurement
 
+class Startup(page_measurement.PageMeasurement):
+  """Performs a measurement of Chromium's startup performance.
+
+  This test must be invoked with either --warm or --cold on the command line. A
+  cold start means none of the Chromium files are in the disk cache. A warm
+  start assumes the OS has already cached much of Chromium's content. For warm
+  tests, you should repeat the page set to ensure it's cached.
+  """
 
-class StartupWarm(page_measurement.PageMeasurement):
-  """Test how long Chrome takes to load when warm."""
   HISTOGRAMS_TO_RECORD = {
     'messageloop_start_time' :
         'Startup.BrowserMessageLoopStartTimeFromMainEntry',
@@ -16,10 +22,27 @@ class StartupWarm(page_measurement.PageMeasurement):
     'open_tabs_time' : 'Startup.BrowserOpenTabs'}
 
   def __init__(self):
-    super(StartupWarm, self).__init__(needs_browser_restart_after_each_run=True,
-                                      discard_first_result=True)
+    super(Startup, self).__init__(needs_browser_restart_after_each_run=True)
+    self._cold = False
+
+  def AddCommandLineOptions(self, parser):
+    parser.add_option('--cold', action='store_true',
+                      help='Clear the OS disk cache before performing the test')
+    parser.add_option('--warm', action='store_true',
+                      help='Start up with everything already cached')
 
   def CustomizeBrowserOptions(self, options):
+    # TODO: Once the bots start running benchmarks, enforce that either --warm
+    # or --cold is explicitly specified.
+    # assert options.warm != options.cold, \
+    #     "You must specify either --warm or --cold"
+    self._cold = options.cold
+
+    if self._cold:
+      options.clear_sytem_cache_for_browser_and_profile_on_start = True
+    else:
+      self.discard_first_result = True
+
     options.AppendExtraBrowserArg('--enable-stats-collection-bindings')
 
     # Old commandline flags used for reference builds.
@@ -35,7 +58,6 @@ class StartupWarm(page_measurement.PageMeasurement):
         'statsCollectionController :'
         'domAutomationController).getBrowserHistogram("%s")')
 
-
     for display_name, histogram_name in self.HISTOGRAMS_TO_RECORD.iteritems():
       result = tab.EvaluateJavaScript(get_histogram_js % histogram_name)
       result = json.loads(result)

tools/perf/run_measurement

@@ -78,8 +78,8 @@ def main():
     "scrolling_benchmark": "smoothness",
     "smoothness_benchmark": "smoothness",
     "smoothness_measurement": "smoothness",
-    "startup_benchmark": "startup_warm",
-    "startup_measurement": "startup_warm",
+    "startup_benchmark": "startup_warm_blank_page",
+    "startup_measurement": "startup",
     "tab_switching_measurement": "tab_switching",
   }
 

tools/telemetry/telemetry/core/platform/desktop_platform_backend.py

@@ -22,7 +22,7 @@ class DesktopPlatformBackend(platform_backend.PlatformBackend):
     flush_command_mtime = 0
 
     chrome_root = util.GetChromiumSrcDir()
-    for build_type, build_dir in util.BuildDirectoryIterator():
+    for build_dir, build_type in util.GetBuildDirectories():
       candidate = os.path.join(chrome_root, build_dir, build_type,
                                self.GetFlushUtilityName())
       if os.access(candidate, os.X_OK):
@@ -43,9 +43,12 @@ class DesktopPlatformBackend(platform_backend.PlatformBackend):
     args = [flush_command, '--recurse']
     directory_contents = os.listdir(directory)
     for item in directory_contents:
-      if item not in ignoring:
+      if not ignoring or item not in ignoring:
         args.append(os.path.join(directory, item))
 
+    if len(args) < 3:
+      return
+
     p = subprocess.Popen(args)
     p.wait()
     assert p.returncode == 0, 'Failed to flush system cache'

tools/telemetry/telemetry/page/page_test.py

@@ -71,6 +71,10 @@ class PageTest(object):
     the first run of the test can warm things up. """
     return self._discard_first_result
 
+  @discard_first_result.setter
+  def discard_first_result(self, discard):
+    self._discard_first_result = discard
+
   @property
   def clear_cache_before_each_run(self):
     """When set to True, the browser's disk and memory cache will be cleared