tools/cygprofile: Add custom URLs and scrolling to profile_android_startup.

Allows to collect profiling data from several page loads, and to scroll
the screen while doing so.

Bug: 758566
Change-Id: I4f0bb8fc167e69dea349dbd4b5ee733193996e41
Reviewed-on: https://chromium-review.googlesource.com/801016
Commit-Queue: Benoit L <lizeb@chromium.org>
Reviewed-by: Egor Pasko <pasko@chromium.org>
Reviewed-by: Matthew Cary <mattcary@chromium.org>
Cr-Commit-Position: refs/heads/master@{#522392}

tools/android/native_lib_memory/process_residency.py

@@ -84,16 +84,19 @@ def PlotResidency(data, output_filename):
     data: (dict) As returned by ParseDump().
     output_filename: (str) Output filename.
   """
+  residency = data['residency']
+  max_percentage = max((100. * sum(d)) / len(d) for d in residency.values())
+  logging.info('Max residency = %.2f%%', max_percentage)
+
   start = data['start']
   end = data['end']
-  data = data['residency']
   fig, ax = plt.subplots(figsize=(20, 10))
-  timestamps = sorted(data.keys())
-  x_max = len(data.values()[0]) * 4096
+  timestamps = sorted(residency.keys())
+  x_max = len(residency.values()[0]) * 4096
   for t in timestamps:
     offset_ms = (t - timestamps[0]) / 1e6
-    incore = [i * 4096 for (i, x) in enumerate(data[t]) if x]
-    outcore = [i * 4096 for (i, x) in enumerate(data[t]) if not x]
+    incore = [i * 4096 for (i, x) in enumerate(residency[t]) if x]
+    outcore = [i * 4096 for (i, x) in enumerate(residency[t]) if not x]
     percentage = 100. * len(incore) / (len(incore) + len(outcore))
     plt.text(x_max, offset_ms, '%.1f%%' % percentage)
     for (d, color) in ((incore, (.2, .6, .05, 1)), (outcore, (1, 0, 0, 1))):

tools/cygprofile/orderfile_generator_backend.py

@@ -14,10 +14,10 @@ Example usage:
     --target-arch=arm
 """
 
+import argparse
 import hashlib
 import json
 import logging
-import optparse
 import os
 import re
 import shutil
@@ -460,8 +460,22 @@ class OrderfileGenerator(object):
         self._BUILD_ROOT, self._options.arch + '_uninstrumented_out')
 
     if options.profile:
+      output_directory = os.path.join(self._instrumented_out_dir, 'Release')
+      host_cyglog_dir = os.path.join(output_directory, 'cyglog_data')
+      # Only override the defaults when using lightweight instrumentation,
+      # as the regular profiling code is likely too slow for these.
+      urls = [profile_android_startup.AndroidProfileTool.TEST_URL]
+      use_wpr = True
+      simulate_user = False
+      if options.simulate_user and not options.lightweight_instrumentation:
+        logging.error(
+            '--simulate-user required --lightweight-instrumentation, ignoring.')
+      if options.lightweight_instrumentation:
+        urls = options.urls
+        use_wpr = not options.no_wpr
+        simulate_user = options.simulate_user
       self._profiler = profile_android_startup.AndroidProfileTool(
-          os.path.join(self._instrumented_out_dir, 'Release'))
+          output_directory, host_cyglog_dir, use_wpr, urls, simulate_user)
 
     self._output_data = {}
     self._step_recorder = StepRecorder(options.buildbot)
@@ -742,47 +756,49 @@ class OrderfileGenerator(object):
     return self._output_data
 
 
-def CreateOptionParser():
-  parser = optparse.OptionParser()
-  parser.add_option(
+def CreateArgumentParser():
+  """Creates and returns the argument parser."""
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
       '--lightweight-instrumentation', action='store_true', default=False,
       help='Use the lightweight instrumentation path')
-  parser.add_option(
+  parser.add_argument(
       '--buildbot', action='store_true',
       help='If true, the script expects to be run on a buildbot')
-  parser.add_option(
+  parser.add_argument(
       '--verify', action='store_true',
       help='If true, the script only verifies the current orderfile')
-  parser.add_option('--target-arch', action='store', dest='arch',
-                    default=cygprofile_utils.DetectArchitecture(),
-                    choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
-                    help='The target architecture for which to build')
-  parser.add_option('--output-json', action='store', dest='json_file',
-                    help='Location to save stats in json format')
-  parser.add_option(
+  parser.add_argument('--target-arch', action='store', dest='arch',
+                      default=cygprofile_utils.DetectArchitecture(),
+                      choices=['arm', 'arm64', 'x86', 'x86_64', 'x64', 'mips'],
+                      help='The target architecture for which to build')
+  parser.add_argument('--output-json', action='store', dest='json_file',
+                      help='Location to save stats in json format')
+  parser.add_argument(
       '--skip-profile', action='store_false', dest='profile', default=True,
       help='Don\'t generate a profile on the device. Only patch from the '
       'existing profile.')
-  parser.add_option(
+  parser.add_argument(
       '--skip-patch', action='store_false', dest='patch', default=True,
       help='Only generate the raw (unpatched) orderfile, don\'t patch it.')
-  parser.add_option(
+  parser.add_argument(
       '--netrc', action='store',
       help='A custom .netrc file to use for git checkin. Only used on bots.')
-  parser.add_option(
+  parser.add_argument(
       '--branch', action='store', default='master',
       help='When running on buildbot with a netrc, the branch orderfile '
       'hashes get checked into.')
   # Note: -j50 was causing issues on the bot.
-  parser.add_option(
+  parser.add_argument(
       '-j', '--jobs', action='store', default=20,
       help='Number of jobs to use for compilation.')
-  parser.add_option(
+  parser.add_argument(
       '-l', '--max-load', action='store', default=4, help='Max cpu load.')
-  parser.add_option('--goma-dir', help='GOMA directory.')
-  parser.add_option(
+  parser.add_argument('--goma-dir', help='GOMA directory.')
+  parser.add_argument(
       '--use-goma', action='store_true', help='Enable GOMA.', default=False)
-  parser.add_option('--adb-path', help='Path to the adb binary.')
+  parser.add_argument('--adb-path', help='Path to the adb binary.')
+  profile_android_startup.AddProfileCollectionArguments(parser)
   return parser
 
 
@@ -815,11 +831,11 @@ def CreateOrderfile(options, orderfile_updater_class):
   return False
 
 
-def main(argv):
-  parser = CreateOptionParser()
-  options, _ = parser.parse_args(argv)
+def main():
+  parser = CreateArgumentParser()
+  options = parser.parse_args()
   return 0 if CreateOrderfile(options, OrderfileUpdater) else 1
 
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv))
+  sys.exit(main())

tools/cygprofile/profile_android_startup.py

@@ -16,7 +16,6 @@ import os
 import shutil
 import subprocess
 import sys
-import tempfile
 import time
 
 _SRC_PATH = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
@@ -61,6 +60,21 @@ def _DownloadFromCloudStorage(bucket, sha1_file_name):
     raise Exception('Exception executing command %s' % ' '.join(cmd))
 
 
+def _SimulateSwipe(device, x1, y1, x2, y2):
+  """Simulates a swipe on a device from (x1, y1) to (x2, y2).
+
+  Coordinates are in (device dependent) pixels, and the origin is at the upper
+  left corner.
+  The simulated swipe will take 300ms.
+
+  Args:
+    device: (device_utils.DeviceUtils) device to run the command on.
+    x1, y1, x2, y2: (int) Coordinates.
+  """
+  args = [str(x) for x in (x1, y1, x2, y2)]
+  device.RunShellCommand(['input', 'swipe'] + args)
+
+
 class WprManager(object):
   """A utility to download a WPR archive, host it, and forward device ports to
   it.
@@ -109,7 +123,7 @@ class WprManager(object):
     self._host_https_port = ports['https']
 
   def _StopWpr(self):
-    """ Stop the WPR and forwarder. """
+    """ Stop the WPR and forwarder."""
     print 'Stopping WPR on host...'
     if self._wpr_server:
       self._wpr_server.StopServer()
@@ -168,28 +182,35 @@ class AndroidProfileTool(object):
 
   _DEVICE_CYGLOG_DIR = '/data/local/tmp/chrome/cyglog'
 
-  # TEST_URL must be a url in the WPR_ARCHIVE.
-  _TEST_URL = 'https://www.google.com/#hl=en&q=science'
+  TEST_URL = 'https://www.google.com/#hl=en&q=science'
   _WPR_ARCHIVE = os.path.join(
       os.path.dirname(__file__), 'memory_top_10_mobile_000.wprgo')
 
-  # TODO(jbudorick): Make host_cyglog_dir mandatory after updating
-  # downstream clients. See crbug.com/639831 for context.
-  def __init__(self, output_directory, host_cyglog_dir=None):
+  def __init__(self, output_directory, host_cyglog_dir, use_wpr, urls,
+               simulate_user):
+    """Constructor.
+
+    Args:
+      output_directory: (str) Chrome build directory.
+      host_cyglog_dir: (str) Where to store the profiles.
+      use_wpr: (bool) Whether to use Web Page Replay.
+      urls: (str) URLs to load. Have to be contained in the WPR archive if
+                  use_wpr is True.
+      simulate_user: (bool) Whether to simulate a user.
+    """
     devices = device_utils.DeviceUtils.HealthyDevices()
     self._device = devices[0]
     self._cygprofile_tests = os.path.join(
         output_directory, 'cygprofile_unittests')
-    self._host_cyglog_dir = host_cyglog_dir or os.path.join(
-        output_directory, 'cyglog_data')
+    self._host_cyglog_dir = host_cyglog_dir
+    self._use_wpr = use_wpr
+    self._urls = urls
+    self._simulate_user = simulate_user
     self._SetUpDevice()
 
   def RunCygprofileTests(self):
     """Run the cygprofile unit tests suite on the device.
 
-    Args:
-      path_to_tests: The location on the host machine with the compiled
-                     cygprofile test binary.
     Returns:
       The exit code for the tests.
     """
@@ -223,18 +244,12 @@ class AndroidProfileTool(object):
     try:
       changer = self._SetChromeFlags(package_info)
       self._SetUpDeviceFolders()
-      # Start up chrome once with a blank page, just to get the one-off
-      # activities out of the way such as apk resource extraction and profile
-      # creation.
-      self._StartChrome(package_info, 'about:blank')
-      time.sleep(15)
-      self._KillChrome(package_info)
-      self._SetUpDeviceFolders()
-      with WprManager(self._WPR_ARCHIVE, self._device,
-                      package_info.cmdline_file, package_info.package):
-        self._StartChrome(package_info, self._TEST_URL)
-        time.sleep(90)
-        self._KillChrome(package_info)
+      if self._use_wpr:
+        with WprManager(self._WPR_ARCHIVE, self._device,
+                        package_info.cmdline_file, package_info.package):
+          self._RunProfileCollection(package_info, self._simulate_user)
+      else:
+        self._RunProfileCollection(package_info, self._simulate_user)
     finally:
       self._RestoreChromeFlags(changer)
 
@@ -242,6 +257,39 @@ class AndroidProfileTool(object):
     self._DeleteDeviceData()
     return data
 
+  def _RunProfileCollection(self, package_info, simulate_user):
+    """Runs the profile collection tasks.
+
+    If |simulate_user| is True, then try to simulate a real user, with swiping.
+    Also do a first load of the page instead of about:blank, in order to
+    exercise the cache. This is not desirable with a page that only contains
+    cachable resources, as in this instance the network code will not be called.
+
+    Args:
+      package_info: Which Chrome package to use.
+      simulate_user: (bool) Whether to try to simulate a user interacting with
+                     the browser.
+    """
+    initial_url = self._urls[0] if simulate_user else 'about:blank'
+    # Start up chrome once with a page, just to get the one-off
+    # activities out of the way such as apk resource extraction and profile
+    # creation.
+    self._StartChrome(package_info, initial_url)
+    time.sleep(15)
+    self._KillChrome(package_info)
+    self._SetUpDeviceFolders()
+    for url in self._urls:
+      self._StartChrome(package_info, url)
+      time.sleep(15)
+      if simulate_user:
+        # Down, down, up, up.
+        _SimulateSwipe(self._device, 200, 700, 200, 300)
+        _SimulateSwipe(self._device, 200, 700, 200, 300)
+        _SimulateSwipe(self._device, 200, 700, 200, 1000)
+        _SimulateSwipe(self._device, 200, 700, 200, 1000)
+      time.sleep(30)
+      self._KillChrome(package_info)
+
   def Cleanup(self):
     """Delete all local and device files left over from profiling. """
     self._DeleteDeviceData()
@@ -249,10 +297,9 @@ class AndroidProfileTool(object):
 
   def _Install(self, apk):
     """Installs Chrome.apk on the device.
+
     Args:
       apk: The location of the chrome apk to profile.
-      package_info: A PackageInfo structure describing the chrome apk,
-                    as from pylib/constants.
     """
     print 'Installing apk...'
     self._device.Install(apk)
@@ -287,7 +334,7 @@ class AndroidProfileTool(object):
       changer.Restore()
 
   def _SetUpDeviceFolders(self):
-    """Creates folders on the device to store cyglog data. """
+    """Creates folders on the device to store cyglog data."""
     print 'Setting up device folders...'
     self._DeleteDeviceData()
     self._device.RunShellCommand(
@@ -349,7 +396,19 @@ class AndroidProfileTool(object):
     return [os.path.join(cyglog_dir, x) for x in files]
 
 
-def main():
+def AddProfileCollectionArguments(parser):
+  """Adds the profiling collection arguments to |parser|."""
+  parser.add_argument(
+      '--no-wpr', action='store_true', help='Don\'t use WPR.')
+  parser.add_argument('--urls', type=str, help='URLs to load.',
+                      default=[AndroidProfileTool.TEST_URL],
+                      nargs='+')
+  parser.add_argument(
+      '--simulate-user', action='store_true', help='More realistic collection.')
+
+
+def CreateArgumentParser():
+  """Creates and return the argument parser."""
   parser = argparse.ArgumentParser()
   parser.add_argument(
       '--adb-path', type=os.path.realpath,
@@ -364,7 +423,12 @@ def main():
       '--trace-directory', type=os.path.realpath,
       help='Directory in which cyglog traces will be stored. '
            'Defaults to <output-directory>/cyglog_data')
+  AddProfileCollectionArguments(parser)
+  return parser
+
 
+def main():
+  parser = CreateArgumentParser()
   args = parser.parse_args()
 
   devil_chromium.Initialize(
@@ -380,7 +444,8 @@ def main():
     raise Exception('Unable to determine package info for %s' % args.apk_path)
 
   profiler = AndroidProfileTool(
-      args.output_directory, host_cyglog_dir=args.trace_directory)
+      args.output_directory, host_cyglog_dir=args.trace_directory,
+      use_wpr=not args.no_wpr, urls=args.urls, simulate_user=args.simulate_user)
   profiler.CollectProfile(args.apk_path, package_info)
   return 0