Add perf tests for starting Chrome with URL

This makes the --url option generally available, and adds tests for
starting Chrome with a URL, with and without a simultaneously
restoring a session.

Note:- As part of this I have added to telemetry the ability
to specify different browser options for each page, and to
specify the startup URL in the page set (this could easily
be extended to other browser options). This was needed to
avoid having to create 6 separate benchmarks to test
different combinations of URLs and sessions to be restored.

BUG=298052

Review URL: https://codereview.chromium.org/113563004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@243849 0039d316-1c4b-4281-b951-d872f2087c98

tools/perf/benchmarks/session_restore_with_url.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry import test
+
+from measurements import session_restore_with_url
+
+
+class SessionRestoreWithUrlCold(test.Test):
+  """Measure Chrome cold session restore with startup URLs."""
+  tag = 'cold'
+  test = session_restore_with_url.SessionRestoreWithUrl
+  page_set = 'page_sets/startup_pages.json'
+  options = {'cold': True,
+             'pageset_repeat_iters': 5}
+
+class SessionRestoreWithUrlWarm(test.Test):
+  """Measure Chrome warm session restore with startup URLs."""
+  tag = 'warm'
+  test = session_restore_with_url.SessionRestoreWithUrl
+  page_set = 'page_sets/startup_pages.json'
+  options = {'warm': True,
+             'pageset_repeat_iters': 10}
+

tools/perf/benchmarks/start_with_url.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from telemetry import test
+
+from measurements import startup
+
+
+class StartWithUrlCold(test.Test):
+  """Measure time to start Chrome cold with startup URLs"""
+  tag = 'cold'
+  test = startup.Startup
+  page_set = 'page_sets/startup_pages.json'
+  options = {'cold': True,
+             'pageset_repeat_iters': 5}
+
+class StartWithUrlWarm(test.Test):
+  """Measure time to start Chrome warm with startup URLs"""
+  tag = 'warm'
+  test = startup.Startup
+  page_set = 'page_sets/startup_pages.json'
+  options = {'warm': True,
+             'pageset_repeat_iters': 10}
+

tools/perf/measurements/image_decoding.py

@@ -19,7 +19,7 @@ class ImageDecoding(page_measurement.PageMeasurement):
     """)
     tab.StartTimelineRecording()
 
-  def NeedsBrowserRestartAfterEachRun(self, browser):
+  def StopBrowserAfterPage(self, browser, page):
     return not browser.tabs[0].ExecuteJavaScript("""
         window.chrome &&
             chrome.gpuBenchmarking &&

tools/perf/measurements/session_restore.py

@@ -38,7 +38,8 @@ class SessionRestore(startup.Startup):
     # Reject any pageset that contains more than one WPR archive.
     wpr_archives = {}
     for page in page_set:
-      wpr_archives[page_set.WprFilePathForPage(page)] = True
+      if not page.is_local:
+        wpr_archives[page_set.WprFilePathForPage(page)] = True
 
     if len(wpr_archives.keys()) > 1:
       raise Exception("Invalid pageset: more than 1 WPR archive found.: " +

tools/perf/measurements/session_restore_with_url.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from measurements import session_restore
+
+
+class SessionRestoreWithUrl(session_restore.SessionRestore):
+
+  def __init__(self):
+    super(SessionRestoreWithUrl, self).__init__()
+
+  def CanRunForPage(self, page):
+    # Run for every page in the page set that has a startup url.
+    return page.hasattr('startup_url')

tools/perf/measurements/startup.py

@@ -15,7 +15,8 @@ class Startup(page_measurement.PageMeasurement):
   """
 
   def __init__(self):
-    super(Startup, self).__init__(needs_browser_restart_after_each_run=True)
+    super(Startup, self).__init__(needs_browser_restart_after_each_run=True,
+                                  action_name_to_run='navigate_steps')
 
   def AddCommandLineOptions(self, parser):
     parser.add_option('--cold', action='store_true',

tools/perf/measurements/startwithurl.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-from measurements import startup
-from metrics import startup_metric
-
-class StartWithUrl(startup.Startup):
-  """Measures Chromium's startup performance when started with a URL
-
-  This test inherits support for the --warm or --cold command line options -
-  see startup.py for details.
-  """
-
-  def __init__(self):
-    super(StartWithUrl, self).__init__()
-    self.close_tabs_before_run = False
-
-
-  def AddCommandLineOptions(self, parser):
-    super(StartWithUrl, self).AddCommandLineOptions(parser)
-    parser.add_option('--url', action='store', default=None,
-                      help='Start with a request to open a specific URL')
-
-  def CustomizeBrowserOptions(self, options):
-    super(StartWithUrl, self).CustomizeBrowserOptions(options)
-    if options.url:
-      browser_options = options.browser_options
-      browser_options.startup_url = options.url
-    options.AppendExtraBrowserArgs([
-        '--restore-last-session'
-    ])
-
-  def CanRunForPage(self, page):
-    # No matter how many pages in the pageset, just perform one test iteration.
-    return page.page_set.pages.index(page) == 0
-
-  def RunNavigateSteps(self, page, tab):
-    # Overriden so that no page navigation occurs.
-    pass
-
-  def ValidatePageSet(self, page_set):
-    # Reject any pageset that contains more than one WPR archive.
-    wpr_archives = {}
-    for page in page_set:
-      wpr_archives[page_set.WprFilePathForPage(page)] = True
-
-    if len(wpr_archives.keys()) > 1:
-      raise Exception("Invalid pageset: more than 1 WPR archive found.: " +
-          ', '.join(wpr_archives.keys()))
-
-  def MeasurePage(self, page, tab, results):
-    # Wait for all tabs to finish loading.
-    for i in xrange(len(tab.browser.tabs)):
-      t = tab.browser.tabs[i]
-      t.WaitForDocumentReadyStateToBeComplete()
-
-    startup_metric.StartupMetric().AddResults(tab, results)
-

tools/perf/page_sets/data/startup_pages.json

+{
+    "description": "Describes the Web Page Replay archives for a page set. Don't edit by hand! Use record_wpr for updating.", 
+    "archives": {
+        "startup_pages_004.wpr": [
+            "http://news.bbc.co.uk", 
+            "http://kapook.com"
+        ]
+    }
+}
\ No newline at end of file

tools/perf/page_sets/startup_pages.json

+{
+  "description": [ 
+  "Pages for testing starting Chrome with a URL. ",
+  "Note that this file can't be used with record_wpr, since record_wpr ",
+  "requires a true navigate step, which we do not want for startup testing.",
+  "Instead use record_wpr startup_pages_record to record data for this test."],
+  "archive_data_file": "data/startup_pages.json",
+  "pages": [
+    {
+      "url": "about:blank",
+      "why": "typical page",
+      "navigate_steps" : [
+        {"action": "set_startup_url", "startup_url": "about:blank"},
+        {"action": "wait", "seconds": 10 }
+      ]
+    },
+    {
+      "url": "http://news.bbc.co.uk",
+      "why": "typical page",
+      "navigate_steps" : [
+        {"action": "set_startup_url", "startup_url": "http://news.bbc.co.uk"},
+        {"action": "wait", "seconds": 10 }
+      ]
+    },
+    {
+      "url": "http://kapook.com",
+      "why": "Horribly complex page - stress test!",
+      "navigate_steps" : [
+        {"action": "set_startup_url", "startup_url": "http://kapook.com"},
+        {"action": "wait", "seconds": 10 }
+      ]
+    }
+  ]
+}

tools/perf/page_sets/startup_pages_record.json

+{
+  "description": 
+  ["Pages to record data for testing starting Chrome with a URL. We can't use",
+   "startup_pages.json with record_wpr, since record_wpr requires a default", 
+   "navigate step, which we don't want for startup testing; but we do want to",
+   "record the pages it uses.",
+   "Also, record_wpr fails on about:blank, which we want to include in startup", 
+   "testing."],
+  "archive_data_file": "data/startup_pages.json",
+  "pages": [
+    {
+      "url": "http://news.bbc.co.uk",
+      "why": "typical page"
+    },
+    {
+      "url": "http://kapook.com",
+      "why": "Horribly complex page - stress test!"
+    }
+  ]
+}
\ No newline at end of file

tools/telemetry/telemetry/page/actions/gesture_action.py

@@ -30,7 +30,7 @@ class GestureAction(page_action.PageAction):
   def RunGesture(self, page, tab, previous_action):
     raise NotImplementedError()
 
-  def CustomizeBrowserOptions(self, options):
+  def CustomizeBrowserOptionsForPageSet(self, options):
     options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
 
   def GetActiveRangeOnTimeline(self, timeline):

tools/telemetry/telemetry/page/actions/page_action.py

@@ -24,9 +24,25 @@ class PageAction(object):
     self._timeline_marker_base_name = None
     self._timeline_marker_id = None
 
-  def CustomizeBrowserOptions(self, options):
+  def CustomizeBrowserOptionsForPageSet(self, options):
     """Override to add action-specific options to the BrowserOptions
-    object."""
+    object. These options will be set for the whole page set.
+
+    If the browser is not being restarted for every page in the page set then
+    all browser options required for the action must be set here. This, however,
+    requires that they do not conflict with options require by other actions
+    used up by the page set.
+    """
+    pass
+
+  def CustomizeBrowserOptionsForSinglePage(self, options):
+    """Override to add action-specific options to the BrowserOptions
+    object. These options will be set for just the page calling the action
+
+    This will only take effect if the browser is restarted for the page calling
+    the action, so should only be used in tests that restart the browser for
+    each page.
+    """
     pass
 
   def WillRunAction(self, page, tab):

tools/telemetry/telemetry/page/actions/set_startup_url.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from telemetry.page.actions import page_action
+
+class SetStartupUrlAction(page_action.PageAction):
+  """ Sets the URL to be loaded when the browser starts
+
+  This action sets a URL to be added to the command line or Android Intent when
+  the browser is started. It only really makes sense for tests that restart
+  the browser for each page.
+  """
+
+  def RunAction(self, page, tab, previous_action):
+    pass
+
+  def CustomizeBrowserOptionsForSinglePage(self, options):
+    assert hasattr(self,'startup_url')
+    options.browser_options.startup_url = self.startup_url

tools/telemetry/telemetry/page/gtest_test_results.py

@@ -63,6 +63,8 @@ class GTestTestResults(page_test_results.PageTestResults):
     super(GTestTestResults, self).addSkip(test, reason)
     test_name = GTestTestResults._formatTestname(test)
     logging.warning('===== SKIPPING TEST %s: %s =====', test_name, reason)
+    if self._timestamp == None:
+      self._timestamp = time.time()
     print >> self._output_stream, '[       OK ]', test_name, (
         '(%0.f ms)' % self._GetMs())
     sys.stdout.flush()

tools/telemetry/telemetry/page/page.py

@@ -57,7 +57,7 @@ class Page(object):
   @property
   def is_local(self):
     """Returns True iff this URL is local. This includes chrome:// URLs."""
-    return self._scheme == 'file' or self._scheme == 'chrome'
+    return self._scheme in ['file', 'chrome', 'about']
 
   @property
   def file_path(self):

tools/telemetry/telemetry/page/page_runner.py

@@ -37,11 +37,13 @@ class _RunState(object):
     self.profiler_dir = None
     self.repeat_state = None
 
-  def StartBrowser(self, test, page_set, page, possible_browser,
+  def StartBrowserIfNeeded(self, test, page_set, page, possible_browser,
                    credentials_path, archive_path):
     started_browser = not self.browser
     # Create a browser.
     if not self.browser:
+      test.CustomizeBrowserOptionsForSinglePage(page,
+                                                possible_browser.finder_options)
       self.browser = possible_browser.Create()
       self.browser.credentials.credentials_path = credentials_path
 
@@ -211,7 +213,11 @@ def _PrepareAndRunPage(test, page_set, expectations, finder_options,
   tries = 3
   while tries:
     try:
-      state.StartBrowser(test, page_set, page, possible_browser,
+      if test.RestartBrowserBeforeEachPage():
+        state.StopBrowser()
+        # If we are restarting the browser for each page customize the per page
+        # options for just the current page before starting the browser.
+      state.StartBrowserIfNeeded(test, page_set, page, possible_browser,
                          credentials_path, page.archive_path)
 
       expectation = expectations.GetExpectationForPage(state.browser, page)
@@ -237,7 +243,7 @@ def _PrepareAndRunPage(test, page_set, expectations, finder_options,
       if finder_options.profiler:
         state.StopProfiling()
 
-      if test.NeedsBrowserRestartAfterEachRun(state.browser):
+      if (test.StopBrowserAfterPage(state.browser, page)):
         state.StopBrowser()
 
       break
@@ -298,8 +304,8 @@ def Run(test, page_set, expectations, finder_options):
   if page_set.user_agent_type:
     browser_options.browser_user_agent_type = page_set.user_agent_type
 
-  for page in pages:
-    test.CustomizeBrowserOptionsForPage(page, possible_browser.finder_options)
+  test.CustomizeBrowserOptionsForPageSet(page_set,
+                                         possible_browser.finder_options)
   if finder_options.profiler:
     profiler_class = profiler_finder.FindProfiler(finder_options.profiler)
     profiler_class.CustomizeBrowserOptions(possible_browser.browser_type,

tools/telemetry/telemetry/page/page_test.py

@@ -111,10 +111,26 @@ class PageTest(object):
   def close_tabs_before_run(self, close_tabs):
     self._close_tabs_before_run = close_tabs
 
-  def NeedsBrowserRestartAfterEachRun(self, browser):  # pylint: disable=W0613
-    """Override to specify browser restart after each run."""
+  def RestartBrowserBeforeEachPage(self):
+    """ Should the browser be restarted for the page?
+
+    This returns true if the test needs to unconditionally restart the
+    browser for each page. It may be called before the browser is started.
+    """
     return self._needs_browser_restart_after_each_run
 
+  def StopBrowserAfterPage(self, browser, page):  # pylint: disable=W0613
+    """Should the browser be stopped after the page is run?
+
+    This is called after a page is run to decide whether the browser needs to
+    be stopped to clean up its state. If it is stopped, then it will be
+    restarted to run the next page.
+
+    A test that overrides this can look at both the page and the browser to
+    decide whether it needs to stop the browser.
+    """
+    return False
+
   def AddCommandLineOptions(self, parser):
     """Override to expose command-line options for this test.
 
@@ -127,14 +143,32 @@ class PageTest(object):
     """Override to add test-specific options to the BrowserOptions object"""
     pass
 
-  def CustomizeBrowserOptionsForPage(self, page, options):
-    """Add options specific to the test and the given page."""
-    if not self.CanRunForPage(page):
-      return
+  def CustomizeBrowserOptionsForPageSet(self, page_set, options):
+    """Set options required for this page set.
+
+    These options will be used every time the browser is started while running
+    this page set. They may, however, be further modified by
+    CustomizeBrowserOptionsForSinglePage or by the profiler.
+    """
+    for page in page_set:
+      if not self.CanRunForPage(page):
+        return
+      interactive = options and options.interactive
+      for action in GetCompoundActionFromPage(
+          page, self._action_name_to_run, interactive):
+        action.CustomizeBrowserOptionsForPageSet(options)
+
+  def CustomizeBrowserOptionsForSinglePage(self, page, options):
+    """Set options specific to the test and the given page.
+
+    This will be called with the current page when the browser is (re)started.
+    Changing options at this point only makes sense if the browser is being
+    restarted for each page.
+    """
     interactive = options and options.interactive
     for action in GetCompoundActionFromPage(
         page, self._action_name_to_run, interactive):
-      action.CustomizeBrowserOptions(options)
+      action.CustomizeBrowserOptionsForSinglePage(options)
 
   def WillStartBrowser(self, browser):
     """Override to manipulate the browser environment before it launches."""

tools/telemetry/telemetry/page/record_wpr.py

@@ -38,10 +38,11 @@ class RecordPage(page_test.PageTest):
   def CanRunForPage(self, page):
     return page.url.startswith('http')
 
-  def CustomizeBrowserOptionsForPage(self, page, options):
-    for compound_action in self._CompoundActionsForPage(page, options):
-      for action in compound_action:
-        action.CustomizeBrowserOptions(options)
+  def CustomizeBrowserOptionsForPageSet(self, pset, options):
+    for page in pset:
+      for compound_action in self._CompoundActionsForPage(page, options):
+        for action in compound_action:
+          action.CustomizeBrowserOptionsForPageSet(options)
 
   def WillNavigateToPage(self, _, tab):
     """Override to ensure all resources are fetched from network."""