android: Cleanup stack trace symbolizer

This CL performs a little cleanup of the content of
third_party/android_platform/development/scripts/:

- Add ToolPath() and GetAaptPath() methods to
  pylib.constants.host_paths, and remove the old
  versions from symbol.py, adjusting all callers.

  Note that ToolPath() now takes a second argument
  to specify the target CPU architecture, instead of
  relying on a hidden global variable.

  + Add new unit-test (host_paths_unittest.py).

- Remove a few unused global variables in symbol.py,
  and document the few remaining one (especially that
  they are liberally modified from elsewhere!).

  Also make a few functions private by adding an
  underscore prefix to them.

  Remove Uname() which was completely unused.

  Remove FindToolchain() since its result was
  completely ignored.

  Add an optional cpu_arch parameter to
  SymbolInformationForSet(), and make asan_symbolize.py,
  its only user, pass a corresponding argument.

- Cleanup asan_symbolize.py a little, and add an
  optional command-line argument --arch=ARCH to
  specify the target CPU architecture (default
  is 'arm', as in the previous implementation).

- Remove CallCppFilt() from symbol.py and rewrite it
  directly into tools/cygprofile/symbol_extractor.py,
  its only user. Also run the unit tests correctly.

BUG=NONE
R=agrieve@chromium.org,lizeb@chromium.org,egeunis@chromium.org

Change-Id: If172662461a2047d7485eedd4658149283fd1ca7
Reviewed-on: https://chromium-review.googlesource.com/1004635Reviewed-by: agrieve <agrieve@chromium.org>
Reviewed-by: Egor Pasko <pasko@chromium.org>
Commit-Queue: David Turner <digit@chromium.org>
Cr-Commit-Position: refs/heads/master@{#549836}

build/android/PRESUBMIT.py

@@ -61,6 +61,7 @@ def CommonChecks(input_api, output_api):
           J('gyp', 'util', 'build_utils_test.py'),
           J('gyp', 'util', 'md5_check_test.py'),
           J('play_services', 'update_test.py'),
+          J('pylib', 'constants', 'host_paths_unittest.py'),
           J('pylib', 'gtest', 'gtest_test_instance_test.py'),
           J('pylib', 'instrumentation',
             'instrumentation_test_instance_test.py'),

build/android/asan_symbolize.py

@@ -23,17 +23,23 @@ with host_paths.SysPath(
 
 _RE_ASAN = re.compile(r'(.*?)(#\S*?)\s+(\S*?)\s+\((.*?)\+(.*?)\)')
 
+# This named tuple models a parsed Asan log line.
+AsanParsedLine = collections.namedtuple('AsanParsedLine',
+                                        'prefix,library,pos,rel_address')
+
+# This named tuple models an Asan log line. 'raw' is the raw content
+# while 'parsed' is None or an AsanParsedLine instance.
+AsanLogLine = collections.namedtuple('AsanLogLine', 'raw,parsed')
+
 def _ParseAsanLogLine(line):
+  """Parse line into corresponding AsanParsedLine value, if any, or None."""
   m = re.match(_RE_ASAN, line)
   if not m:
     return None
-  return {
-      'prefix': m.group(1),
-      'library': m.group(4),
-      'pos': m.group(2),
-      'rel_address': '%08x' % int(m.group(5), 16),
-  }
-
+  return AsanParsedLine(prefix=m.group(1),
+                        library=m.group(4),
+                        pos=m.group(2),
+                        rel_address='%08x' % int(m.group(5), 16))
 
 def _FindASanLibraries():
   asan_lib_dir = os.path.join(host_paths.DIR_SOURCE_ROOT,
@@ -55,38 +61,52 @@ def _TranslateLibPath(library, asan_libs):
   return symbol.TranslateLibPath(library)
 
 
-def _Symbolize(asan_input):
+def _PrintSymbolized(asan_input, arch):
+  """Print symbolized logcat output for Asan symbols.
+
+  Args:
+    asan_input: list of input lines.
+    arch: Target CPU architecture.
+  """
   asan_libs = _FindASanLibraries()
+
+  # Maps library -> [ AsanParsedLine... ]
   libraries = collections.defaultdict(list)
-  asan_lines = []
-  for asan_log_line in [a.rstrip() for a in asan_input]:
-    m = _ParseAsanLogLine(asan_log_line)
-    if m:
-      libraries[m['library']].append(m)
-    asan_lines.append({'raw_log': asan_log_line, 'parsed': m})
 
+  asan_log_lines = []
+  for line in asan_input:
+    line = line.rstrip()
+    parsed = _ParseAsanLogLine(line)
+    if parsed:
+      libraries[parsed.library].append(parsed)
+    asan_log_lines.append(AsanLogLine(raw=line, parsed=parsed))
+
+  # Maps library -> { address -> [(symbol, location, obj_sym_with_offset)...] }
   all_symbols = collections.defaultdict(dict)
+
   for library, items in libraries.iteritems():
     libname = _TranslateLibPath(library, asan_libs)
-    lib_relative_addrs = set([i['rel_address'] for i in items])
+    lib_relative_addrs = set([i.rel_address for i in items])
     # pylint: disable=no-member
     info_dict = symbol.SymbolInformationForSet(libname,
                                                lib_relative_addrs,
-                                               True)
+                                               True,
+                                               target_arch=arch)
     if info_dict:
-      all_symbols[library]['symbols'] = info_dict
-
-  for asan_log_line in asan_lines:
-    m = asan_log_line['parsed']
-    if not m:
-      print asan_log_line['raw_log']
-      continue
-    if (m['library'] in all_symbols and
-        m['rel_address'] in all_symbols[m['library']]['symbols']):
-      s = all_symbols[m['library']]['symbols'][m['rel_address']][0]
-      print '%s%s %s %s' % (m['prefix'], m['pos'], s[0], s[1])
+      all_symbols[library] = info_dict
+
+  for log_line in asan_log_lines:
+    m = log_line.parsed
+    if (m and m.library in all_symbols and
+        m.rel_address in all_symbols[m.library]):
+      # NOTE: all_symbols[lib][address] is a never-emtpy list of tuples.
+      # NOTE: The documentation for SymbolInformationForSet() indicates
+      # that usually one wants to display the last list item, not the first.
+      # The code below takes the first, is this the best choice here?
+      s = all_symbols[m.library][m.rel_address][0]
+      print '%s%s %s %s' % (m.prefix, m.pos, s[0], s[1])
     else:
-      print asan_log_line['raw_log']
+      print log_line.raw
 
 
 def main():
@@ -96,6 +116,8 @@ def main():
                          'Use stdin if not specified.')
   parser.add_option('--output-directory',
                     help='Path to the root build directory.')
+  parser.add_option('--arch', default='arm',
+                    help='CPU architecture name')
   options, _ = parser.parse_args()
 
   if options.output_directory:
@@ -107,7 +129,8 @@ def main():
     asan_input = file(options.logcat, 'r')
   else:
     asan_input = sys.stdin
-  _Symbolize(asan_input.readlines())
+
+  _PrintSymbolized(asan_input.readlines(), options.arch)
 
 
 if __name__ == "__main__":

build/android/pylib/constants/host_paths.py

@@ -6,6 +6,8 @@ import contextlib
 import os
 import sys
 
+from pylib import constants
+
 DIR_SOURCE_ROOT = os.environ.get(
     'CHECKOUT_SOURCE_ROOT',
     os.path.abspath(os.path.join(os.path.dirname(__file__),
@@ -36,3 +38,56 @@ def SysPath(path, position=None):
       sys.path.pop()
     else:
       sys.path.remove(path)
+
+
+# Map of CPU architecture name to (toolchain_name, binprefix) pairs.
+# TODO(digit): Use the build_vars.txt file generated by gn.
+_TOOL_ARCH_MAP = {
+  'arm': ('arm-linux-androideabi-4.9', 'arm-linux-androideabi'),
+  'arm64': ('aarch64-linux-android-4.9', 'aarch64-linux-android'),
+  'x86': ('x86-4.9', 'i686-linux-android'),
+  'x86_64': ('x86_64-4.9', 'x86_64-linux-android'),
+  'x64': ('x86_64-4.9', 'x86_64-linux-android'),
+  'mips': ('mipsel-linux-android-4.9', 'mipsel-linux-android'),
+}
+
+# Cache used to speed up the results of ToolPath()
+# Maps (arch, tool_name) pairs to fully qualified program paths.
+# Useful because ToolPath() is called repeatedly for demangling C++ symbols.
+_cached_tool_paths = {}
+
+
+def ToolPath(tool, cpu_arch):
+  """Return a fully qualifed path to an arch-specific toolchain program.
+
+  Args:
+    tool: Unprefixed toolchain program name (e.g. 'objdump')
+    cpu_arch: Target CPU architecture (e.g. 'arm64')
+  Returns:
+    Fully qualified path (e.g. ..../aarch64-linux-android-objdump')
+  Raises:
+    Exception if the toolchain could not be found.
+  """
+  tool_path = _cached_tool_paths.get((tool, cpu_arch))
+  if tool_path:
+    return tool_path
+
+  toolchain_source, toolchain_prefix = _TOOL_ARCH_MAP.get(
+      cpu_arch, (None, None))
+  if not toolchain_source:
+    raise Exception('Could not find tool chain for ' + cpu_arch)
+
+  toolchain_subdir = (
+      'toolchains/%s/prebuilt/linux-x86_64/bin' % toolchain_source)
+
+  tool_path = os.path.join(constants.ANDROID_NDK_ROOT,
+                           toolchain_subdir,
+                           toolchain_prefix + '-' + tool)
+
+  _cached_tool_paths[(tool, cpu_arch)] = tool_path
+  return tool_path
+
+
+def GetAaptPath():
+  """Returns the path to the 'aapt' executable."""
+  return os.path.join(constants.ANDROID_SDK_TOOLS, 'aapt')

build/android/pylib/constants/host_paths_unittest.py

+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import unittest
+
+import pylib.constants as constants
+import pylib.constants.host_paths as host_paths
+
+
+# This map corresponds to the binprefix of NDK prebuilt toolchains for various
+# target CPU architectures. Note that 'x86_64' and 'x64' are the same.
+_EXPECTED_NDK_TOOL_SUBDIR_MAP = {
+  'arm': 'toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin/' +
+         'arm-linux-androideabi-',
+  'arm64':
+      'toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/' +
+      'aarch64-linux-android-',
+  'x86': 'toolchains/x86-4.9/prebuilt/linux-x86_64/bin/i686-linux-android-',
+  'x86_64':
+      'toolchains/x86_64-4.9/prebuilt/linux-x86_64/bin/x86_64-linux-android-',
+  'x64':
+      'toolchains/x86_64-4.9/prebuilt/linux-x86_64/bin/x86_64-linux-android-',
+   'mips':
+      'toolchains/mipsel-linux-android-4.9/prebuilt/linux-x86_64/bin/' +
+      'mipsel-linux-android-'
+}
+
+
+class HostPathsTest(unittest.TestCase):
+  def setUp(self):
+    logging.getLogger().setLevel(logging.ERROR)
+
+  def test_GetAaptPath(self):
+    _EXPECTED_AAPT_PATH = os.path.join(constants.ANDROID_SDK_TOOLS, 'aapt')
+    self.assertEqual(host_paths.GetAaptPath(), _EXPECTED_AAPT_PATH)
+    self.assertEqual(host_paths.GetAaptPath(), _EXPECTED_AAPT_PATH)
+
+  def test_ToolPath(self):
+    for cpu_arch, binprefix in _EXPECTED_NDK_TOOL_SUBDIR_MAP.iteritems():
+      expected_binprefix = os.path.join(constants.ANDROID_NDK_ROOT, binprefix)
+      expected_path = expected_binprefix + 'foo'
+      self.assertEqual(host_paths.ToolPath('foo', cpu_arch), expected_path)
+
+
+if __name__ == '__main__':
+  unittest.main()

third_party/android_platform/development/scripts/stack.py

@@ -174,7 +174,8 @@ def main(argv):
       symbol.ARCH = value
       arch_defined = True
     elif option == "--chrome-symbols-dir":
-      symbol.CHROME_SYMBOLS_DIR = os.path.join(symbol.CHROME_SRC, value)
+      symbol.CHROME_SYMBOLS_DIR = os.path.join(constants.DIR_SOURCE_ROOT,
+                                               value)
     elif option == "--output-directory":
       constants.SetOutputDirectory(value)
     elif option == "--packed-lib":

tools/cygprofile/cyglog_to_orderfile_unittest.py

@@ -19,9 +19,9 @@ from test_utils import SimpleTestSymbol
 
 sys.path.insert(
     0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
-                    'third_party', 'android_platform', 'development',
-                    'scripts'))
-import symbol
+                    'build', 'android'))
+
+import pylib.constants.host_paths as host_paths
 
 
 # Used for fake demangling on bots where c++filt does not exist.
@@ -44,7 +44,7 @@ class TestObjectFileProcessor(cyglog_to_orderfile.ObjectFileProcessor):
 class TestCyglogToOrderfile(unittest.TestCase):
   def setUp(self):
     self._old_demangle = None
-    if not os.path.exists(symbol.ToolPath('c++filt')):
+    if not os.path.exists(host_paths.ToolPath('c++filt', 'arm')):
       print 'Using fake demangling due to missing c++filt binary'
       self._old_demangle = symbol_extractor.DemangleSymbol
       symbol_extractor.DemangleSymbol = _FakeDemangle

tools/cygprofile/symbol_extractor.py

@@ -15,18 +15,24 @@ import cygprofile_utils
 
 sys.path.insert(
     0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
-                    'third_party', 'android_platform', 'development',
-                    'scripts'))
-import symbol
+                    'build', 'android'))
+
+from pylib import constants
+from pylib.constants import host_paths
 
 _MAX_WARNINGS_TO_PRINT = 200
 
 SymbolInfo = collections.namedtuple('SymbolInfo', ('name', 'offset', 'size',
                                                    'section'))
 
+# Unfortunate global variable :-/
+_arch = 'arm'
+
+
 def SetArchitecture(arch):
   """Set the architecture for binaries to be symbolized."""
-  symbol.ARCH = arch
+  global _arch
+  _arch = arch
 
 
 def _FromObjdumpLine(line):
@@ -89,7 +95,7 @@ def SymbolInfosFromBinary(binary_filename):
   Returns:
     A list of SymbolInfo from the binary.
   """
-  command = (symbol.ToolPath('objdump'), '-t', '-w', binary_filename)
+  command = (host_paths.ToolPath('objdump', _arch), '-t', '-w', binary_filename)
   p = subprocess.Popen(command, shell=False, stdout=subprocess.PIPE)
   try:
     result = _SymbolInfosFromStream(p.stdout)
@@ -116,6 +122,7 @@ def GroupSymbolInfosByOffset(symbol_infos):
     offset_to_symbol_infos[symbol_info.offset].append(symbol_info)
   return dict(offset_to_symbol_infos)
 
+
 def GroupSymbolInfosByName(symbol_infos):
   """Create a dict {name: [symbol_info1, ...], ...}.
 
@@ -132,6 +139,7 @@ def GroupSymbolInfosByName(symbol_infos):
     name_to_symbol_infos[symbol_info.name].append(symbol_info)
   return dict(name_to_symbol_infos)
 
+
 def CreateNameToSymbolInfo(symbol_infos):
   """Create a dict {name: symbol_info, ...}.
 
@@ -161,4 +169,7 @@ def CreateNameToSymbolInfo(symbol_infos):
 
 def DemangleSymbol(mangled_symbol):
   """Return the demangled form of mangled_symbol."""
-  return symbol.CallCppFilt(mangled_symbol)
+  cmd = [host_paths.ToolPath("c++filt", _arch)]
+  process = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+  demangled_symbol, _ = process.communicate(mangled_symbol + '\n')
+  return demangled_symbol