Extension profile generator + benchmark for startup with profile.

Extension profile extender for profile generator and a new benchmark for
startup time on a profile that loads extensions. Also for this purpose, a
script to maintain a static set of extensions in cloud storage, and small
additions to python gsutil wrapper (cloud_storage).

BUG=444230
CQ_EXTRA_TRYBOTS=tryserver.chromium.perf:linux_perf_bisect;tryserver.chromium.perf:mac_perf_bisect

Review URL: https://codereview.chromium.org/1240703003

Cr-Commit-Position: refs/heads/master@{#339807}

tools/perf/benchmarks/start_with_ext.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from core import perf_benchmark
+from measurements import startup
+import page_sets
+from telemetry import benchmark
+
+
+class _StartWithExt(perf_benchmark.PerfBenchmark):
+  """Base benchmark for testing startup with extensions."""
+  page_set = page_sets.BlankPageSetWithExtensionProfile
+  tag = None
+
+  @classmethod
+  def Name(cls):
+    return 'start_with_ext.blank_page'
+
+  @classmethod
+  def ValueCanBeAddedPredicate(cls, _, is_first_result):
+    return not is_first_result
+
+  def SetExtraBrowserOptions(self, options):
+    options.disable_default_apps = False
+
+  def CreatePageTest(self, _):
+    is_cold = (self.tag == 'cold')
+    return startup.Startup(cold=is_cold)
+
+
+@benchmark.Enabled('has tabs')
+@benchmark.Enabled('mac') # Currently only works on mac.
+@benchmark.Disabled('win', 'linux', 'reference', 'android')
+class StartWithExtCold(_StartWithExt):
+  """Measure time to start Chrome cold with extensions."""
+  options = {'pageset_repeat': 5}
+  tag = 'cold'
+
+  @classmethod
+  def Name(cls):
+    return 'start_with_ext.cold.blank_page'
+
+
+@benchmark.Enabled('has tabs')
+@benchmark.Enabled('mac') # Currently only works on mac.
+@benchmark.Disabled('win', 'linux', 'reference', 'android')
+class StartWithExtWarm(_StartWithExt):
+  """Measure time to start Chrome warm with extensions."""
+  options = {'pageset_repeat': 20}
+  tag = 'warm'
+
+  @classmethod
+  def Name(cls):
+    return 'start_with_ext.warm.blank_page'
+

tools/perf/page_sets/blank_page_with_extension_profile.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from page_sets import extension_profile_shared_state
+from telemetry.page import page as page_module
+from telemetry import story
+
+
+class BlankPageWithExtensionProfile(page_module.Page):
+  """A single blank page loaded with a profile with many extensions."""
+
+  def __init__(self, url, page_set):
+    super(BlankPageWithExtensionProfile, self).__init__(
+        url=url, page_set=page_set,
+        shared_page_state_class=extension_profile_shared_state.
+                                ExtensionProfileSharedState)
+
+
+class BlankPageSetWithExtensionProfile(story.StorySet):
+  """PageSet tied to BlankPageWithExtensionProfile."""
+
+  def __init__(self):
+    super(BlankPageSetWithExtensionProfile, self).__init__()
+    self.AddStory(BlankPageWithExtensionProfile(
+        'file://blank_page/blank_page.html', self))

tools/perf/page_sets/extension_profile_shared_state.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import shutil
+
+from profile_creators import extension_profile_extender
+from profile_creators import profile_generator
+from telemetry.page import shared_page_state
+
+
+class ExtensionProfileSharedState(shared_page_state.SharedPageState):
+  """Shared state tied with extension profile.
+
+  Generates extension profile on initialization.
+  """
+
+  def __init__(self, test, finder_options, story_set):
+    super(ExtensionProfileSharedState, self).__init__(
+        test, finder_options, story_set)
+    generator = profile_generator.ProfileGenerator(
+        extension_profile_extender.ExtensionProfileExtender,
+        'extension_profile')
+    self._out_dir = generator.Run(finder_options)
+    if self._out_dir:
+      finder_options.browser_options.profile_dir = self._out_dir
+    else:
+      finder_options.browser_options.dont_override_profile = True
+
+  def TearDownState(self):
+    """Clean up generated profile directory."""
+    super(ExtensionProfileSharedState, self).TearDownState()
+    if self._out_dir:
+      shutil.rmtree(self._out_dir)

tools/perf/profile_creators/extension_profile_extender.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+import time
+import zipfile
+
+from catapult_base import cloud_storage
+from profile_creators import profile_extender
+from telemetry.core import exceptions
+
+
+# Remote target upload directory in cloud storage for extensions.
+REMOTE_DIR = 'extension_set'
+
+# Target zip file.
+ZIP_NAME = 'extensions.zip'
+
+
+class InvalidExtensionArchiveError(exceptions.Error):
+  """Exception thrown when remote archive is invalid or malformed.
+
+  Remote archive should be located at REMOTE_DIR/ZIP_NAME. Upon failure,
+  prompts user to update remote archive using update_remote_extensions
+  script.
+  """
+
+  def __init__(self, msg=''):
+    msg += ('\nTry running\n'
+            '\tpython update_remote_extensions.py -e extension_set.csv\n'
+            'in src/tools/perf/profile_creator subdirectory.')
+    super(InvalidExtensionArchiveError, self).__init__(msg)
+
+
+class ExtensionProfileExtender(profile_extender.ProfileExtender):
+  """Creates a profile with many extensions."""
+
+  def __init__(self, finder_options):
+    super(ExtensionProfileExtender, self).__init__(finder_options)
+    self._extensions = []
+    finder_options.browser_options.disable_default_apps = False
+
+  def Run(self):
+    """Superclass override."""
+    # Download extensions from cloud and force-install extensions into profile.
+    local_extensions_dir = os.path.join(self.profile_path,
+                                        'external_extensions_crx')
+    self._DownloadRemoteExtensions(cloud_storage.PARTNER_BUCKET,
+                                   local_extensions_dir)
+    self._LoadExtensions(local_extensions_dir, self.profile_path)
+
+    try:
+      self.SetUpBrowser()
+      self._WaitForExtensionsToLoad()
+    finally:
+      self.TearDownBrowser()
+
+  def RestrictedOSList(self):
+    """Superclass override."""
+    return ['mac']
+
+  def _DownloadRemoteExtensions(self, remote_bucket, local_extensions_dir):
+    """Downloads and unzips archive of common extensions to disk.
+
+    Args:
+        remote_bucket: bucket to download remote archive from.
+        local_extensions_dir: destination extensions directory.
+
+    Raises:
+        InvalidExtensionArchiveError if remote archive is not found.
+    """
+    remote_zip_path = os.path.join(REMOTE_DIR, ZIP_NAME)
+    local_zip_path = os.path.join(local_extensions_dir, ZIP_NAME)
+    try:
+      cloud_storage.Get(remote_bucket, remote_zip_path, local_zip_path)
+    except:
+      raise InvalidExtensionArchiveError('Can\'t find archive at gs://%s/%s..'
+                                         % (remote_bucket, remote_zip_path))
+    try:
+      with zipfile.ZipFile(local_zip_path, 'r') as extensions_zip:
+        extensions_zip.extractall(local_extensions_dir)
+    finally:
+      os.remove(local_zip_path)
+
+  def _GetExtensionInfoFromCrx(self, crx_file):
+    """Retrieves version + name of extension from CRX archive."""
+    with zipfile.ZipFile(crx_file, 'r') as crx_zip:
+      manifest_contents = crx_zip.read('manifest.json')
+      decoded_manifest = json.loads(manifest_contents)
+      crx_version = decoded_manifest['version']
+      extension_name = decoded_manifest['name']
+    return (crx_version, extension_name)
+
+  def _LoadExtensions(self, local_extensions_dir, profile_dir):
+    """Loads extensions in _local_extensions_dir into user profile.
+
+    Extensions are loaded according to platform specifications at
+    https://developer.chrome.com/extensions/external_extensions.html
+
+    Args:
+        local_extensions_dir: directory containing CRX files.
+        profile_dir: target profile directory for the extensions.
+
+    Raises:
+        InvalidExtensionArchiveError if archive contains a non-CRX file.
+    """
+    ext_files = os.listdir(local_extensions_dir)
+    external_ext_dir = os.path.join(profile_dir, 'External Extensions')
+    os.makedirs(external_ext_dir)
+    for ext_file in ext_files:
+      ext_path = os.path.join(local_extensions_dir, ext_file)
+      if not ext_file.endswith('.crx'):
+        raise InvalidExtensionArchiveError('Archive contains non-crx file %s.'
+                                           % ext_file)
+      (version, name) = self._GetExtensionInfoFromCrx(ext_path)
+      extension_info = {
+        'external_crx': ext_path,
+        'external_version': version,
+        '_comment': name
+      }
+      ext_id = os.path.splitext(os.path.basename(ext_path))[0]
+      extension_json_path = os.path.join(external_ext_dir, '%s.json' % ext_id)
+      with open(extension_json_path, 'w') as f:
+        f.write(json.dumps(extension_info))
+      self._extensions.append(ext_id)
+
+  def _WaitForExtensionsToLoad(self):
+    """Stall until browser has finished installing/loading all extensions."""
+    for extension_id in self._extensions:
+      while True:
+        try:
+          self.browser.extensions.GetByExtensionId(extension_id)
+          break
+        except KeyError:
+          # There's no event signalling when browser finishes installing
+          # or loading an extension so re-check every 5 seconds.
+          time.sleep(5)

tools/perf/profile_creators/extension_set.csv

+# Fields automatically updated by update_remote_extensions.py.
+# Do not manually alter individual fields.
+# To manually add extension, append line:
+#     <extension_name>,<extension_id>,,
+# and run update_remote_extensions.py to update hash and version columns.
+# To manually remove extension, remove the entire line.
+extension_name,id,hash,version
+Avast Online Security,gomekmidlodglbbmalcneegieacbdmki,YTIzNDZlYjE4NWI5NzM4ZTExYTI0MDBhN2RkODg2ZGRiZTE5MTdlYg==,10.2.0.190
+Skype Click to Call,lifbcibllhkdhoafpjfnlhfpfgnpldfl,OWY2MTYwNDg0ZDkyNmNjZDlmMGNlMWUyOWFiNjJmMWMxZTY3N2ZlNw==,7.4.0.9058
+AdBlock,gighmmpiobklfepjocnamgkkbiglidom,NDNlMmEwZTNjMjFjNzk2OTQzMWFkMWYzYmZkM2Q2YjViODA0OTRkZQ==,2.36.2
+Bookmark Manager,gmlllbghnfkpflemihljekbapjopfjik,ZGJlNmRkMDE3N2I2ZTZlZmNjYjE5MTBkY2M5NDJkNTRlMDM3NTQ1Yg==,2.2015.611.10417
+Adblock Plus,cfhdojbkjhnklbpkdaibdccddilifddb,YjJhNjljMWFkNjFhNjA3ODk3OTJhNjVjZmY4ZTJmMzFkN2UyYzU0Nw==,1.9.1
+iLivid,nafaimnnclfjfedmmabolbppcngeolgf,MzdiZGE1NmRjODRlOTZjNmIwYTM0ZDVjYjJhYzcxZGJmOGZiYjg0YQ==,1.1
+SiteAdvisor,fheoggkfdfchfphceeifdbepaooicaho,MzE1YjNhYjg2Mjk0NTYwYTllZGJlYTk5MjMyZjYwZGNmOGNlMGQ2Mw==,4.0.0.0
+Avast SafePrice,eofcbnmajmjmplflapaojjnihcjkigck,MWVlYjBlYWYyZGUzNjQxMWE4NWRkNDczMzRlZWY5Nzc1N2M4Y2M0Yg==,10.2.0.190
+Norton Identity Safe,iikflkcanblccfahdhdonehdalibjnif,OTA1NGI5MThiZGFkOWRkNjAzNGQ2N2EzODdhNzUwNTNkZTIyMjAyZA==,1.0.5
+Application Launcher for Drive (by Google),lmjegmlicamnimmfhcmpkclmigmmcbeh,NWY0NjY2ZDNhNGNlOWU3MTY4MjlhMTE5MWFhZTk4Y2FlZTI5NmUwMQ==,3.2
+Norton Security Toolbar,mkfokfffehpeedafpekjeddnmnjhmcmk,NTdlOWVlZjQ0OTU3MDYyMzlhODEzZDc4N2FiYWIyNDhjMTQxYmI1Yw==,2014.7.12.29
+Ask App for iLivid,mppnoffgpafgpgbaigljliadgbnhljfl,ZTg5YzhhMWIyMDk5MGMzN2ZlNTU5ZGU3MjM5NjMwZTAyZTc1Y2E5NA==,1.1
+Avira Browser Safety,flliilndjeohchalpbbcdekjklbdgfkk,ZTk5NGQ4NzRiNWQ2ZGNmZmE1MWFlMTM3YWYxYzcwZTA1NmI3N2RlYw==,1.4.10
+MSN Homepage & Bing Search Engine,fcfenmboojpjinhpgggodefccipikbpd,NjNkNGVkZDFlNTYzNTBkOGY1ZDFiOWZmZGQyOWIxZDkzOWMwODM0Yg==,0.0.0.8
+Google Cast,boadgeojelhgndaghljhdicfkmllpafd,YjUzYTNhYzA5YWVlZDliZThkZjc1Mzc5Y2U3ZDIyNGRhMWQ4MGJlZQ==,15.623.0.6
+Adobe Acrobat - Create PDF,efaidnbmnnnibpcajpcglclefindmkaj,MjdkNzdjM2MwYzNhMDJmZDRlYTI1ODc3MjZkZGViMDJmMjkzNTFlMw==,15.0.0.0
+Cisco WebEx Extension,jlhmfgmfgeifomenelglieieghnjghma,NmQ0Y2VhMDI0OGU5YzcxNDAxZjdmYWZmYzBjNGUwZjgzYjI0YWZmYg==,1.0.1
+IDM Helper Module,ngpampappnmepgilojfohadhhmbhlaek,YzIzYzUwYmQ3MGNmOGRmMzFhOWE4OGI0NDg0ZWRhNzU1OTIxZGI5Zg==,6.22.3
+Pin It Button,gpdjojdkbbmdfjfahjcgigfpmkopogic,M2Q2NGU0ZTkwZDBhMmI5M2NkYzFkY2I2ZTFiYTFiZjE3ZDVhZjZhMQ==,1.38.2

tools/perf/profile_creators/profile_extender.py

@@ -58,6 +58,17 @@ class ProfileExtender(object):
   def browser(self):
     return self._browser
 
+  def EnabledOSList(self):
+    """Returns a list of OSes that this extender can run on.
+
+    Can be overridden by subclasses.
+
+    Returns:
+        List of OS ('win', 'mac', or 'linux') that this extender can run on.
+        None if this extender can run on all platforms.
+    """
+    return None
+
   def SetUpBrowser(self):
     """Finds and starts the browser.
 
@@ -70,6 +81,13 @@ class ProfileExtender(object):
     """
     possible_browser = self._GetPossibleBrowser(self.finder_options)
 
+    os_name = possible_browser.platform.GetOSName()
+    enabled_os_list = self.EnabledOSList()
+    if enabled_os_list is not None and os_name not in enabled_os_list:
+      raise NotImplementedError(
+        'This profile extender on %s is not yet supported'
+        % os_name)
+
     assert possible_browser.supports_tab_control
     assert (platform.GetHostPlatform().GetOSName() in
         ["win", "mac", "linux"])

tools/perf/profile_creators/update_remote_extensions.py

+# Copyright 2015 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import base64
+import csv
+import json
+import optparse
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+import zipfile
+
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
+    'telemetry'))
+
+from catapult_base import cloud_storage
+from telemetry.core import exceptions
+
+# Remote target upload directory in cloud storage for extensions.
+REMOTE_DIR = 'extension_set'
+
+# Target zip file.
+ZIP_NAME = 'extensions.zip'
+
+
+def _DownloadCrxFromCws(ext_id, dst):
+  """Downloads CRX specified from Chrome Web Store.
+
+  Retrieves CRX (Chrome extension file) specified by ext_id from Chrome Web
+  Store, into directory specified by dst.
+
+  Args:
+      ext_id: id of extension to retrieve.
+      dst: directory to download CRX into
+
+  Returns:
+      Returns local path to downloaded CRX.
+      If download fails, return None.
+  """
+  dst_path = os.path.join(dst, '%s.crx' % ext_id)
+  cws_url = ('https://clients2.google.com/service/update2/crx?response='
+             'redirect&prodversion=38.0&x=id%%3D%s%%26installsource%%3D'
+             'ondemand%%26uc' % ext_id)
+  response = urllib2.urlopen(cws_url)
+  if response.getcode() is not 200:
+    return None
+  with open(dst_path, 'w') as f:
+    f.write(response.read())
+  return dst_path
+
+
+def _UpdateExtensionsInCloud(local_extensions_dir, extensions_csv, remote_dir):
+  """Updates set of extensions in Cloud Storage from a CSV of extension ids.
+
+  From well-formatted CSV file containing some set of extensions
+  (extensions_csv), download them, compress into archive, and update
+  the remote extension archive under REMOTE_DIR in CHROME-PARTNER-TELEMETRY
+  bucket. This script expects 2nd column of CSV file to contain extension ids.
+
+  Args:
+      local_extensions_dir: directory to download CRX files into.
+      extension_csv: CSV to pull extension_ids from.
+      remote_dir: remote directory to put extension archive in cloud storage.
+
+  Raises:
+      Exception if a CRX download fails.
+  """
+
+  # Download CRX to temp files and compress into archive
+  zip_path = os.path.join(local_extensions_dir, ZIP_NAME)
+  extension_zip = zipfile.ZipFile(zip_path, 'w')
+  update_csv = False
+  extensions_info = []
+  with open(extensions_csv, 'rb') as csv_file:
+    reader = csv.reader(csv_file)
+    # Stores comments (in case CSV needs to be updated/rewritten)
+    # and skips header line.
+    comments = []
+    line = ','.join(reader.next())
+    while line.startswith('#'):
+      comments.append(line)
+      line = ','.join(reader.next())
+    # Extract info from CSV.
+    for row in reader:
+      extension_info = {
+        'extension_name': row[0],
+        'id': row[1],
+        'hash': row[2],
+        'version': row[3]
+      }
+
+      print 'Fetching extension %s...' % extension_info['id']
+      crx_path = _DownloadCrxFromCws(extension_info['id'], local_extensions_dir)
+      if crx_path is None:
+        raise exceptions.Error('\tCould not fetch %s.\n\n'
+                               'If this extension dl consistently fails, '
+                               'remove this entry from %s.'
+                               % (extension_info['id'], extensions_csv))
+      (new_hash, new_version) = _CrxHashIfChanged(crx_path, extension_info)
+      if new_hash is not None:
+        update_csv = True
+        extension_info['hash'] =  new_hash
+        extension_info['version'] = new_version
+      extensions_info.append(extension_info)
+      extension_zip.write(crx_path, arcname='%s.crx' % extension_info['id'])
+  extension_zip.close()
+
+  if update_csv:
+    print 'Updating CSV...'
+    _UpdateCsv(comments, extensions_csv, extensions_info)
+
+  print 'Uploading extensions to cloud...'
+  remote_zip_path = os.path.join(remote_dir, ZIP_NAME)
+  cloud_storage.Insert(cloud_storage.PARTNER_BUCKET, remote_zip_path, zip_path)
+
+
+def _CrxHashIfChanged(crx_path, extension_info):
+  """Checks whether downloaded Crx has been altered.
+
+  Compares stored hash with hash of downloaded Crx. If different, alerts user
+  that CRX version has changed and will be updated in CSV file.
+
+  Args:
+    crx_path: Path to downloaded CRX.
+    extension_info: Info from CSV (including id and previous hash) about CRX.
+
+  Returns:
+    New hash and version if extension differed. Otherwise, returns (None, None)
+  """
+  downloaded_hash = _Base64Hash(crx_path)
+  new_version = _GetVersionFromCrx(crx_path)
+  if downloaded_hash != extension_info['hash']:
+    if new_version != extension_info['version']:
+      ans = raw_input('\tWarning: Extension %s version from Web Store differs '
+                      'from CSV version.\n\tIf continued, script will write '
+                      'new hash and version to CSV.\n\tContinue? (y/n) '
+                      % extension_info['id']).lower()
+    else:
+      raise exceptions.Error('Extension %s hash from Web Store differs from '
+                             '\nhash stored in CSV, but versions are the same.')
+    if not ans.startswith('y'):
+      sys.exit('Web Store extension %s hash differs from hash in CSV.'
+               % extension_info['id'])
+    return (downloaded_hash, new_version)
+  return (None, None)
+
+def _UpdateCsv(comments, extensions_csv, extensions_info):
+  """Updates CSV with information in extensions_info.
+
+  Original CSV is overwritten with updated information about each extension.
+  Header comments from original CSV are preserved.
+
+  Args:
+    comments: List containing lines of comments found in header of original CSV.
+    extensions_csv: Path to CSV file.
+    extensions_info: List of extension info to write to CSV. Each entry is
+        a dict containing fields extension_name, id, hash, and version.
+  """
+  # Maintain pre-existing comments.
+  with open(extensions_csv, 'w') as csv_file:
+    csv_file.write('\n'.join(comments))
+    csv_file.write('\n')
+  with open(extensions_csv, 'a') as csv_file:
+    writer = csv.DictWriter(
+        csv_file, fieldnames=['extension_name', 'id', 'hash', 'version'])
+    writer.writeheader()
+    writer.writerows(extensions_info)
+
+def _GetCsvFromArgs():
+  """Parse options to retrieve name of CSV file."""
+  parser = optparse.OptionParser()
+  parser.add_option('-e', '--extension-csv', dest='extension_csv',
+                   help='CSV of extensions to load.')
+  (options, _) = parser.parse_args()
+  if not options.extension_csv:
+    parser.error('Must specify --extension-csv option.')
+  return options.extension_csv
+
+def _GetVersionFromCrx(crx_path):
+  """Retrieves extension version from CRX archive.
+
+  Args:
+    crx_path: path to CRX archive to extract version from.
+  """
+  with zipfile.ZipFile(crx_path, 'r') as crx_zip:
+    manifest_contents = crx_zip.read('manifest.json')
+    version = json.loads(manifest_contents)['version']
+  return version
+
+def _Base64Hash(file_path):
+  return base64.b64encode(cloud_storage.CalculateHash(file_path))
+
+def main():
+  extension_csv = _GetCsvFromArgs()
+  local_extensions_dir = tempfile.mkdtemp()
+  try:
+    _UpdateExtensionsInCloud(local_extensions_dir,
+                             extension_csv, REMOTE_DIR)
+  finally:
+    shutil.rmtree(local_extensions_dir)
+
+if __name__ == '__main__':
+  main()
+