Use llvm symbolizer for stack script.

llvm symbolizer is a more efficient symbolizer than addr2line, objdump,
etc.

In this cl, I 1)created a wrapper instance to interact with
llvm symbolizer, 2)made the stack script to use llvm symbolizer instance,
and then 3)added llvm symbolizer into isolated inputs.

Bug: 774267
Change-Id: I971fb808b97f3a569eb9615f99efa41e3a56f3cb
Reviewed-on: https://chromium-review.googlesource.com/789376Reviewed-by: agrieve <agrieve@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Zhiling Huang <hzl@chromium.org>
Cr-Commit-Position: refs/heads/master@{#521820}

build/secondary/third_party/android_platform/development/scripts/BUILD.gn

@@ -13,9 +13,5 @@ group("stack_py") {
   sources = _py_files
   data = sources
 
-  data += [
-    "${android_tool_prefix}addr2line",
-    "${android_tool_prefix}objdump",
-    "${android_tool_prefix}c++filt",
-  ]
+  data += [ "//third_party/llvm-build/Release+Asserts/bin/llvm-symbolizer" ]
 }

third_party/android_platform/README.chromium

@@ -40,6 +40,10 @@ Added code to capture java stderr for better handling of native->java crashes.
 Fixed invalid using decl in logging header debug.h
 Only attempt to symbolize with ELF libraries.
 
+Changed the stack script to use llvm symbolizer instead of addr2line,
+objdump, etc, since llvm symbolizer is more efficient in finding
+function names, line numbers etc.
+
 Android relocation packing tool details:
     Copy sources from AOSP bionic/tools/relocation_packer
     Remove scripts that regenerate golden test data (not relevant here)

third_party/android_platform/development/scripts/stack

@@ -31,8 +31,13 @@ import sys
 sys.path.insert(0, os.path.join(os.path.dirname(__file__),
                                 os.pardir, os.pardir, os.pardir, os.pardir,
                                 'build', 'android'))
+
 from pylib import constants
 
+sys.path.insert(0, os.path.join(os.path.dirname(__file__),
+                                os.pardir, os.pardir, os.pardir, os.pardir,
+                                'tools', 'python'))
+import llvm_symbolizer
 
 DEFAULT_SYMROOT='/tmp/symbols'
 # From: https://source.android.com/source/build-numbers.html
@@ -225,10 +230,12 @@ def main(argv):
   print ("Reading Android symbols from: "
          + os.path.normpath(symbol.SYMBOLS_DIR))
   chrome_search_path = symbol.GetLibrarySearchPaths()
-  print ("Searching for Chrome symbols from within: "
-         + ':'.join((os.path.normpath(d) for d in chrome_search_path)))
-  stack_core.ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome,
-                          arch_defined)
+
+  with llvm_symbolizer.LLVMSymbolizer() as symbolizer:
+    print ("Searching for Chrome symbols from within: "
+           + ':'.join((os.path.normpath(d) for d in chrome_search_path)))
+    stack_core.ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome,
+                            arch_defined, symbolizer)
 
   if rootdir:
     # be a good citizen and clean up...os.rmdir and os.removedirs() don't work

third_party/android_platform/development/scripts/stack_core.py

@@ -158,7 +158,7 @@ def PrintDivider():
   print
   print '-----------------------------------------------------\n'
 
-def ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome, arch_defined):
+def ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome, arch_defined, llvm_symbolizer):
   """Convert strings containing native crash to a stack."""
   InitWidthRelatedLineMatchers()
 
@@ -189,7 +189,7 @@ def ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome, arch_define
       print ('Find ABI:' + arch)
       symbol.ARCH = arch
 
-  ResolveCrashSymbol(list(useful_log), more_info)
+  ResolveCrashSymbol(list(useful_log), more_info, llvm_symbolizer)
   end = time.time()
   logging.debug('Finished resolving symbols. Elapsed time: %.4fs',
                 (end - start))
@@ -304,7 +304,7 @@ class PreProcessLog:
         useful_log.append(line)
     return useful_log, self._so_dirs
 
-def ResolveCrashSymbol(lines, more_info):
+def ResolveCrashSymbol(lines, more_info, llvm_symbolizer):
   """Convert unicode strings which contains native crash to a stack
   """
 
@@ -313,39 +313,6 @@ def ResolveCrashSymbol(lines, more_info):
   last_frame = -1
   pid = -1
 
-  # It is faster to get symbol information with a single call rather than with
-  # separate calls for each line. Since symbol.SymbolInformation caches results,
-  # we can extract all the addresses that we will want symbol information for
-  # from the log and call symbol.SymbolInformation so that the results are
-  # cached in the following lookups.
-  code_addresses = {}
-
-  # Collects all java exception lines, keyed by pid for later output during
-  # native crash handling.
-  java_stderr_by_pid = {}
-  for line in lines:
-    lib, address = None, None
-
-    match = _TRACE_LINE.match(line) or _DEBUG_TRACE_LINE.match(line)
-    if match:
-      address, lib = match.group('address', 'lib')
-
-    match = _VALUE_LINE.match(line)
-    if match and not _CODE_LINE.match(line):
-      (_0, _1, address, lib, _2, _3) = match.groups()
-
-    if lib:
-      code_addresses.setdefault(lib, set()).add(address)
-
-    java_stderr_match = _JAVA_STDERR_LINE.search(line)
-    if java_stderr_match:
-      pid, msg = java_stderr_match.groups()
-      java_stderr_by_pid.setdefault(pid, []).append(msg)
-
-  for lib in code_addresses:
-    symbol.SymbolInformationForSet(
-        symbol.TranslateLibPath(lib), code_addresses[lib], more_info)
-
   for line in lines:
     # AndroidFeedback adds zero width spaces into its crash reports. These
     # should be removed or the regular expresssions will fail to match.
@@ -410,25 +377,19 @@ def ResolveCrashSymbol(lines, more_info):
         logging.debug('Identified lib: %s' % area)
         # If a calls b which further calls c and c is inlined to b, we want to
         # display "a -> b -> c" in the stack trace instead of just "a -> c"
-        info = symbol.SymbolInformation(area, code_addr, more_info)
+        # To use llvm symbolizer, the hexadecimal address has to start with 0x.
+        info = llvm_symbolizer.GetSymbolInformation(
+            os.path.join(symbol.SYMBOLS_DIR, symbol.TranslateLibPath(area)),
+            '0x' + code_addr)
         logging.debug('symbol information: %s' % info)
         nest_count = len(info) - 1
-        for (source_symbol, source_location, object_symbol_with_offset) in info:
-          if not source_symbol:
-            if symbol_present:
-              source_symbol = symbol.CallCppFilt(symbol_name)
-            else:
-              source_symbol = UNKNOWN
-          if not source_location:
-            source_location = area
+        for source_symbol, source_location in info:
           if nest_count > 0:
             nest_count = nest_count - 1
             trace_lines.append(('v------>', source_symbol, source_location))
           else:
-            if not object_symbol_with_offset:
-              object_symbol_with_offset = source_symbol
             trace_lines.append((code_addr,
-                                object_symbol_with_offset,
+                                source_symbol,
                                 source_location))
     match = _VALUE_LINE.match(line)
     if match:
@@ -436,20 +397,14 @@ def ResolveCrashSymbol(lines, more_info):
       if area == UNKNOWN or area == HEAP or area == STACK or not area:
         value_lines.append((addr, value, '', area))
       else:
-        info = symbol.SymbolInformation(area, value, more_info)
-        (source_symbol, source_location, object_symbol_with_offset) = info.pop()
-        if not source_symbol:
-          if symbol_present:
-            source_symbol = symbol.CallCppFilt(symbol_name)
-          else:
-            source_symbol = UNKNOWN
-        if not source_location:
-          source_location = area
-        if not object_symbol_with_offset:
-          object_symbol_with_offset = source_symbol
+        info = llvm_symbolizer.GetSymbolInformation(
+            os.path.join(symbol.SYMBOLS_DIR, symbol.TranslateLibPath(area)),
+            '0x' + code_addr)
+        source_symbol, source_location = info.pop()
+
         value_lines.append((addr,
                             value,
-                            object_symbol_with_offset,
+                            source_symbol,
                             source_location))
 
   java_lines = []

tools/python/llvm_symbolizer.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+import re
+import subprocess
+import threading
+
+_CHROME_SRC = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)
+_LLVM_SYMBOLIZER_PATH = os.path.join(
+    _CHROME_SRC, 'third_party', 'llvm-build', 'Release+Asserts', 'bin',
+    'llvm-symbolizer')
+
+_BINARY = re.compile(r'0b[0,1]+')
+_HEX = re.compile(r'0x[0-9,a-e]+')
+_OCTAL = re.compile(r'0[0-7]+')
+
+_UNKNOWN = '<UNKNOWN>'
+
+
+def _CheckValidAddr(addr):
+  """
+  Check whether the addr is valid input to llvm symbolizer.
+  Valid addr has to be octal, binary, or hex number.
+
+  Args:
+    addr: addr to be entered to llvm symbolizer.
+
+  Returns:
+    whether the addr is valid input to llvm symbolizer.
+  """
+  return _HEX.match(addr) or _OCTAL.match(addr) or _BINARY.match(addr)
+
+
+class LLVMSymbolizer(object):
+  def __init__(self):
+    """Create a LLVMSymbolizer instance that interacts with the llvm symbolizer.
+
+    The purpose of the LLVMSymbolizer is to get function names and line
+    numbers of an address from the symbols library.
+    """
+    self._llvm_symbolizer_subprocess = None
+    # Allow only one thread to call GetSymbolInformation at a time.
+    self._lock = threading.Lock()
+
+  def Start(self):
+    """Start the llvm symbolizer subprocess.
+
+    Create a subprocess of the llvm symbolizer executable, which will be used
+    to retrieve function names etc.
+    """
+    if os.path.isfile(_LLVM_SYMBOLIZER_PATH):
+      self._llvm_symbolizer_subprocess = subprocess.Popen(
+        [_LLVM_SYMBOLIZER_PATH], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
+    else:
+      logging.error('Cannot find llvm_symbolizer here: %s.' %
+                    _LLVM_SYMBOLIZER_PATH)
+      self._llvm_symbolizer_subprocess = None
+
+  def Close(self):
+    """Close the llvm symbolizer subprocess.
+
+    Close the subprocess by closing stdin, stdout and killing the subprocess.
+    """
+    with self._lock:
+      if self._llvm_symbolizer_subprocess:
+        self._llvm_symbolizer_subprocess.kill()
+        self._llvm_symbolizer_subprocess = None
+
+  def __enter__(self):
+    """Start the llvm symbolizer subprocess."""
+    self.Start()
+    return self
+
+  def __exit__(self, exc_type, exc_val, exc_tb):
+    """Close the llvm symbolizer subprocess."""
+    self.Close()
+
+  def GetSymbolInformation(self, lib, addr):
+    """Return the corresponding function names and line numbers.
+
+    Args:
+      lib: library to search for info.
+      addr: address to look for info.
+
+    Returns:
+      A list of (function name, line numbers) tuple.
+    """
+    if (self._llvm_symbolizer_subprocess is None or not lib
+        or not _CheckValidAddr(addr) or not os.path.isfile(lib)):
+      return [(_UNKNOWN, lib)]
+
+    with self._lock:
+      self._llvm_symbolizer_subprocess.stdin.write('%s %s\n' % (lib, addr))
+      self._llvm_symbolizer_subprocess.stdin.flush()
+
+      result = []
+      # Read till see new line, which is a symbol of end of output.
+      # One line of function name is always followed by one line of line number.
+      while True:
+        line = self._llvm_symbolizer_subprocess.stdout.readline()
+        if line != '\n':
+          line_numbers = self._llvm_symbolizer_subprocess.stdout.readline()
+          result.append(
+            (line[:-1],
+             line_numbers[:-1]))
+        else:
+          return result