Telemetry: Build symfs for Android VTune profiler

Extend and use android_profiling_helper.py to set up a symfs for VTune.
Queries the required libraries from the profiler's output and pulls them from
the device. This allows VTune to resolve all symbols in the profiling output.

BUG=

Review URL: https://codereview.chromium.org/308123002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@276740 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/core/platform/profiler/android_profiling_helper.py

@@ -9,6 +9,7 @@ import os
 import platform
 import re
 import shutil
+import sqlite3
 import subprocess
 
 from telemetry.core import util
@@ -109,6 +110,27 @@ def GetRequiredLibrariesForPerfProfile(profile_file):
   return libs
 
 
+def GetRequiredLibrariesForVTuneProfile(profile_file):
+  """Returns the set of libraries necessary to symbolize a given VTune profile.
+
+  Args:
+    profile_file: Path to VTune profile to analyse.
+
+  Returns:
+    A set of required library file names.
+  """
+  db_file = os.path.join(profile_file, 'sqlite-db', 'dicer.db')
+  conn = sqlite3.connect(db_file)
+
+  try:
+    # The 'dd_module_file' table lists all libraries on the device. Only the
+    # ones with 'bin_located_path' are needed for the profile.
+    query = 'SELECT bin_path, bin_located_path FROM dd_module_file'
+    return set(row[0] for row in conn.cursor().execute(query) if row[1])
+  finally:
+    conn.close()
+
+
 def CreateSymFs(device, symfs_dir, libraries, use_symlinks=True):
   """Creates a symfs directory to be used for symbolizing profiles.
 

tools/telemetry/telemetry/core/platform/profiler/android_profiling_helper_unittest.py

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 import glob
 import os
+import pickle
 import re
 import shutil
 import tempfile
@@ -62,6 +63,37 @@ class TestAndroidProfilingHelper(tab_test_case.TabTestCase):
     finally:
       android_profiling_helper.subprocess = real_subprocess
 
+  @test.Enabled('android')
+  def testGetRequiredLibrariesForVTuneProfile(self):
+    vtune_db_output = os.path.join(
+        util.GetUnittestDataDir(), 'sample_vtune_db_output')
+    with open(vtune_db_output, 'rb') as f:
+      vtune_db_output = pickle.load(f)
+
+    mock_cursor = simple_mock.MockObject()
+    mock_cursor.ExpectCall(
+        'execute').WithArgs(simple_mock.DONT_CARE).WillReturn(vtune_db_output)
+
+    mock_conn = simple_mock.MockObject()
+    mock_conn.ExpectCall('cursor').WillReturn(mock_cursor)
+    mock_conn.ExpectCall('close')
+
+    mock_sqlite3 = simple_mock.MockObject()
+    mock_sqlite3.ExpectCall(
+        'connect').WithArgs(simple_mock.DONT_CARE).WillReturn(mock_conn)
+
+    real_sqlite3 = android_profiling_helper.sqlite3
+    android_profiling_helper.sqlite3 = mock_sqlite3
+    try:
+      libs = android_profiling_helper.GetRequiredLibrariesForVTuneProfile('foo')
+      self.assertEqual(libs, set([
+          '/data/app-lib/com.google.android.apps.chrome-1/libchrome.2019.0.so',
+          '/system/lib/libdvm.so',
+          '/system/lib/libc.so',
+          '/system/lib/libm.so']))
+    finally:
+      android_profiling_helper.sqlite3 = real_sqlite3
+
   @test.Enabled('android')
   def testCreateSymFs(self):
     # pylint: disable=W0212

tools/telemetry/telemetry/core/platform/profiler/vtune_profiler.py

@@ -8,30 +8,24 @@ import subprocess
 import sys
 import tempfile
 
-from telemetry.core import util
 from telemetry.core.platform import profiler
+from telemetry.core.platform.profiler import android_profiling_helper
 
 
 class _SingleProcessVTuneProfiler(object):
   """An internal class for using vtune for a given process."""
-  def __init__(self, pid, output_file, platform_backend):
+  def __init__(self, pid, output_file, browser_backend, platform_backend):
     self._pid = pid
+    self._browser_backend = browser_backend
     self._platform_backend = platform_backend
     self._output_file = output_file
     self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
     cmd = ['amplxe-cl', '-collect', 'hotspots',
            '-target-pid', str(pid), '-r', self._output_file]
-    if platform_backend.GetOSName() == 'android':
+    self._is_android = platform_backend.GetOSName() == 'android'
+    if self._is_android:
       cmd += ['-target-system', 'android']
 
-      print 'On Android, assuming $CHROMIUM_OUT_DIR/Release/lib has a fresh'
-      print 'symbolized library matching the one on device.'
-      search_dir = os.path.join(util.GetChromiumSrcDir(),
-                                os.environ.get('CHROMIUM_OUT_DIR', 'out'),
-                                os.environ.get('BUILDTYPE', 'Release'),
-                                'lib')
-      cmd += ['-search-dir', search_dir]
-
     self._proc = subprocess.Popen(
         cmd, stdout=self._tmp_output_file, stderr=subprocess.STDOUT)
 
@@ -54,9 +48,29 @@ class _SingleProcessVTuneProfiler(object):
     finally:
       self._tmp_output_file.close()
 
-    if not exit_code:
-      print 'To view the profile, run:'
-      print '  amplxe-gui %s' % self._output_file
+    if exit_code:
+      # The renderer process was swapped out. Now that we made sure VTune has
+      # stopped, return without further processing the invalid profile.
+      return self._output_file
+
+    if self._is_android:
+      required_libs = \
+          android_profiling_helper.GetRequiredLibrariesForVTuneProfile(
+              self._output_file)
+
+      device = self._browser_backend.adb.device()
+      symfs_root = os.path.dirname(self._output_file)
+      android_profiling_helper.CreateSymFs(device,
+                                           symfs_root,
+                                           required_libs,
+                                           use_symlinks=True)
+      logging.info('Resolving symbols in profile.')
+      subprocess.call(['amplxe-cl', '-finalize', '-r', self._output_file,
+                       '-search-dir', symfs_root])
+
+    print 'To view the profile, run:'
+    print '  amplxe-gui %s' % self._output_file
+
     return self._output_file
 
   def _GetStdOut(self):
@@ -90,7 +104,8 @@ class VTuneProfiler(profiler.Profiler):
         continue
 
       self._process_profilers.append(
-          _SingleProcessVTuneProfiler(pid, output_file, platform_backend))
+          _SingleProcessVTuneProfiler(pid, output_file, browser_backend,
+                                      platform_backend))
 
   @classmethod
   def name(cls):

tools/telemetry/unittest_data/sample_vtune_db_output

+(lp0
+(V/data/app-lib/com.google.android.apps.chrome-1/libchrome.2019.0.so
+p1
+V/tmp/amplxe-tmp-dominikg/modules.android/libchrome.2019.0.so/00a9b3533743e080ab94e760bc891ab8/libchrome.2019.0.so
+p2
+tp3
+a(V/system/bin/app_process
+p4
+Ntp5
+a(V/system/bin/linker
+p6
+Ntp7
+a(V/system/lib/libcutils.so
+p8
+Ntp9
+a(V/system/lib/liblog.so
+p10
+Ntp11
+a(V/system/lib/libc.so
+p12
+V/tmp/amplxe-tmp-dominikg/modules.android/libc.so/abc82d050be774f342adeb7e91fe5a0a/libc.so
+p13
+tp14
+a(V/system/lib/libstdc++.so
+p15
+Ntp16
+a(V/system/lib/libm.so
+p17
+V/tmp/amplxe-tmp-dominikg/modules.android/libm.so/55c87bc955f6c7bf84cce8522fb1e88b/libm.so
+p18
+tp19
+a(V/system/lib/libutils.so
+p20
+Ntp21
+a(V/system/lib/libcorkscrew.so
+p22
+Ntp23
+a(V/system/lib/libgccdemangle.so
+p24
+Ntp25
+a(V/system/lib/libz.so
+p26
+Ntp27
+a(V/system/lib/libbinder.so
+p28
+Ntp29
+a(V/system/lib/libandroid_runtime.so
+p30
+Ntp31
+a(V/system/lib/libandroidfw.so
+p32
+Ntp33
+a(V/system/lib/libskia.so
+p34
+Ntp35
+a(V/system/lib/libemoji.so
+p36
+Ntp37
+a(V/system/lib/libjpeg.so
+p38
+Ntp39
+a(V/system/lib/libexpat.so
+p40
+Ntp41
+a(V/system/lib/libEGL.so
+p42
+Ntp43
+a(V/system/lib/libGLES_trace.so
+p44
+Ntp45
+a(V/system/lib/libstlport.so
+p46
+Ntp47
+a(V/system/lib/libGLESv2.so
+p48
+Ntp49
+a(V/system/lib/libnativehelper.so
+p50
+Ntp51
+a(V/system/lib/libnetutils.so
+p52
+Ntp53
+a(V/system/lib/libui.so
+p54
+Ntp55
+a(V/system/lib/libhardware.so
+p56
+Ntp57
+a(V/system/lib/libsync.so
+p58
+Ntp59
+a(V/system/lib/libgui.so
+p60
+Ntp61
+a(V/system/lib/libcamera_client.so
+p62
+Ntp63
+a(V/system/lib/libcamera_metadata.so
+p64
+Ntp65
+a(V/system/lib/libsqlite.so
+p66
+Ntp67
+a(V/system/lib/libicuuc.so
+p68
+Ntp69
+a(V/system/lib/libgabi++.so
+p70
+Ntp71
+a(V/system/lib/libicui18n.so
+p72
+Ntp73
+a(V/system/lib/libdvm.so
+p74
+V/tmp/amplxe-tmp-dominikg/modules.android/libdvm.so/8f6c9167e20b377dc90646a1bac0e201/libdvm.so
+p75
+tp76
+a(V/system/lib/libselinux.so
+p77
+Ntp78
+a(V/system/lib/libGLESv1_CM.so
+p79
+Ntp80
+a(V/system/lib/libETC1.so
+p81
+Ntp82
+a(V/system/lib/libhardware_legacy.so
+p83
+Ntp84
+a(V/system/lib/libwpa_client.so
+p85
+Ntp86
+a(V/system/lib/libsonivox.so
+p87
+Ntp88
+a(V/system/lib/libcrypto.so
+p89
+Ntp90
+a(V/system/lib/libssl.so
+p91
+Ntp92
+a(V/system/lib/libmedia.so
+p93
+Ntp94
+a(V/system/lib/libstagefright_foundation.so
+p95
+Ntp96
+a(V/system/lib/libaudioutils.so
+p97
+Ntp98
+a(V/system/lib/libspeexresampler.so
+p99
+Ntp100
+a(V/system/lib/libaudioresample.so
+p101
+Ntp102
+a(V/system/lib/libusbhost.so
+p103
+Ntp104
+a(V/system/lib/libharfbuzz_ng.so
+p105
+Ntp106
+a(V/system/lib/libhwui.so
+p107
+Ntp108
+a(V/system/lib/libRS.so
+p109
+Ntp110
+a(V/system/lib/libbcc.so
+p111
+Ntp112
+a(V/system/lib/libbcinfo.so
+p113
+Ntp114
+a(V/system/lib/libLLVM.so
+p115
+Ntp116
+a(V/system/lib/libRScpp.so
+p117
+Ntp118
+a(V/system/lib/libjavacore.so
+p119
+Ntp120
+a(V/system/lib/libdrmframework_jni.so
+p121
+Ntp122
+a(V/system/lib/libdrmframework.so
+p123
+Ntp124
+a(V/system/lib/libstagefright_yuv.so
+p125
+Ntp126
+a(V/system/lib/libdrm.so
+p127
+Ntp128
+a(V/system/lib/libmedia_jni.so
+p129
+Ntp130
+a(V/system/lib/libstagefright_omx.so
+p131
+Ntp132
+a(V/system/lib/libvorbisidec.so
+p133
+Ntp134
+a(V/system/lib/libva.so
+p135
+Ntp136
+a(V/system/lib/libva-android.so
+p137
+Ntp138
+a(V/system/lib/libva-tpi.so
+p139
+Ntp140
+a(V/system/lib/libva_videodecoder.so
+p141
+Ntp142
+a(V/system/lib/libmixvbp.so
+p143
+Ntp144
+a(V/system/lib/libasfparser.so
+p145
+Ntp146
+a(V/system/lib/libstagefright_enc_common.so
+p147
+Ntp148
+a(V/system/lib/libstagefright_avc_common.so
+p149
+Ntp150
+a(V/system/lib/libmultidisplay.so
+p151
+Ntp152
+a(V/system/lib/libsepdrm.so
+p153
+Ntp154
+a(V/system/lib/libmtp.so
+p155
+Ntp156
+a(V/system/lib/libexif.so
+p157
+Ntp158
+a(V/system/lib/libstagefright_amrnb_common.so
+p159
+Ntp160
+a(V/system/lib/libexif_jni.so
+p161
+Ntp162
+a(V/system/lib/libsoundpool.so
+p163
+Ntp164
+a(V/system/lib/libvideoeditor_jni.so
+p165
+Ntp166
+a(V/system/lib/libaudioflinger.so
+p167
+Ntp168
+a(V/system/lib/libcommon_time_client.so
+p169
+Ntp170
+a(V/system/lib/libnbaio.so
+p171
+Ntp172
+a(V/system/lib/libeffects.so
+p173
+Ntp174
+a(V/system/lib/libpowermanager.so
+p175
+Ntp176
+a(V/system/lib/libvideoeditor_core.so
+p177
+Ntp178
+a(V/system/lib/libvideoeditor_osal.so
+p179
+Ntp180
+a(V/system/lib/libvideoeditor_videofilters.so
+p181
+Ntp182
+a(V/system/lib/libvideoeditorplayer.so
+p183
+Ntp184
+a(V/system/lib/libsharedbuffer.so
+p185
+Ntp186
+a(V/system/lib/libva_videoencoder.so
+p187
+Ntp188
+a(V/system/lib/libintelmetadatabuffer.so
+p189
+Ntp190
+a(V/system/lib/librs_jni.so
+p191
+Ntp192
+a(V/system/lib/libandroid.so
+p193
+Ntp194
+a(V/system/lib/libharfbuzz.so
+p195
+Ntp196
+a(V/system/lib/libstagefright.so
+p197
+Ntp198
+a(V/system/lib/libwebcore.so
+p199
+Ntp200
+a(V/system/lib/libchromium_net.so
+p201
+Ntp202
+a(V/data/app-lib/com.google.android.apps.chrome-1/libchromium_android_linker.so
+p203
+Ntp204
+a(V/system/lib/libjnigraphics.so
+p205
+Ntp206
+a(V/dev/ashmem/dalvik-jit-code-cache (deleted)
+p207
+Ntp208
+a.
\ No newline at end of file