Fix stack tool for library in apk crashes

This patches the python zipfile implementation to ignore struct unpack errors
in _decodeExtra. See http://bugs.python.org/issue14315

Also adds a --verbose option that enables a bunch of logging to help diagnose
failed symbolization.

Fixes stack tool for gn builds where libs are put in the top-level output
directory.

Review URL: https://codereview.chromium.org/1006603002

Cr-Commit-Position: refs/heads/master@{#325573}

third_party/android_platform/README.chromium

@@ -20,6 +20,11 @@ along with the files required to build it in the chromium tree.
 
 Local Modifications:
 Only picked the few scripts needed by chrome.
+
+The scripts have been modified to better suit Chromium development. Changes
+include, but are not limited to, the following:
+Added memoization of addr2line and objdump.
+Added option to change the amount of symbolization done.
 Updated output directories to use environment variable.
 When calling addr2line, check the symbol is a file (and not a directory).
 Added support for parsing LOG(FATAL) and DCHECK errors and their
@@ -27,6 +32,7 @@ Added support for parsing LOG(FATAL) and DCHECK errors and their
 Added support for finding symbols when library is loaded directly from the APK.
 Changed the toolchain to remove references to 4.6 toolchains.
 Added support for arch=x64 as an alias to arch=x86_64
+Added debug logging and --verbose parameter.
 
 Android relocation packing tool details:
     Copy sources from AOSP bionic/tools/relocation_packer

third_party/android_platform/development/scripts/stack

@@ -18,6 +18,7 @@
 
 import getopt
 import glob
+import logging
 import os
 import sys
 
@@ -57,6 +58,9 @@ def PrintUsage():
   print "  --arch=arm|arm64|x64|x86|mips"
   print "       the target architecture"
   print
+  print "  --verbose"
+  print "       enable extra logging, particularly for debugging failed symbolization"
+  print
   print "  FILE should contain a stack trace in it somewhere"
   print "       the tool will find that and re-print it with"
   print "       source files and line numbers.  If you don't"
@@ -108,15 +112,16 @@ def UnzipSymbols(symbolfile, symdir=None):
     return (symdir, symdir)
 
 
-def main():
+def main(argv):
   try:
-    options, arguments = getopt.getopt(sys.argv[1:], "",
+    options, arguments = getopt.getopt(argv, "",
                                        ["more-info",
                                         "less-info",
                                         "chrome-symbols-dir=",
                                         "symbols-dir=",
                                         "symbols-zip=",
                                         "arch=",
+                                        "verbose",
                                         "help"])
   except getopt.GetoptError, unused_error:
     PrintUsage()
@@ -138,6 +143,8 @@ def main():
       more_info = True
     elif option == "--less-info":
       more_info = False
+    elif option == "--verbose":
+      logging.basicConfig(level=logging.DEBUG)
 
   if len(arguments) > 1:
     PrintUsage()
@@ -167,6 +174,6 @@ def main():
     os.system(cmd)
 
 if __name__ == "__main__":
-  main()
+  sys.exit(main(sys.argv[1:]))
 
 # vi: ts=2 sw=2

third_party/android_platform/development/scripts/stack_core.py

@@ -16,6 +16,7 @@
 
 """stack symbolizes native crash dumps."""
 
+import logging
 import re
 
 import symbol
@@ -192,6 +193,7 @@ def ConvertTrace(lines, more_info):
     if match:
       frame, code_addr, area, symbol_present, symbol_name = match.group(
           'frame', 'address', 'lib', 'symbol_present', 'symbol_name')
+      logging.debug('Found trace line: %s' % line.strip())
 
       if frame <= last_frame and (trace_lines or value_lines):
         PrintOutput(trace_lines, value_lines, more_info)
@@ -203,9 +205,11 @@ def ConvertTrace(lines, more_info):
       if area == UNKNOWN or area == HEAP or area == STACK:
         trace_lines.append((code_addr, "", area))
       else:
+        logging.debug('Identified lib: %s' % area)
         # If a calls b which further calls c and c is inlined to b, we want to
         # display "a -> b -> c" in the stack trace instead of just "a -> c"
         info = symbol.SymbolInformation(area, code_addr, more_info)
+        logging.debug('symbol information: %s' % info)
         nest_count = len(info) - 1
         for (source_symbol, source_location, object_symbol_with_offset) in info:
           if not source_symbol:

third_party/android_platform/development/scripts/symbol.py

@@ -21,8 +21,10 @@ The information can include symbol names, offsets, and source locations.
 
 import glob
 import itertools
+import logging
 import os
 import re
+import struct
 import subprocess
 import zipfile
 
@@ -36,6 +38,19 @@ ARCH = "arm"
 
 TOOLCHAIN_INFO = None
 
+# See:
+# http://bugs.python.org/issue14315
+# https://hg.python.org/cpython/rev/6dd5e9556a60#l2.8
+def PatchZipFile():
+  oldDecodeExtra = zipfile.ZipInfo._decodeExtra
+  def decodeExtra(self):
+    try:
+      oldDecodeExtra(self)
+    except struct.error:
+      pass
+  zipfile.ZipInfo._decodeExtra = decodeExtra
+PatchZipFile()
+
 def Uname():
   """'uname' for constructing prebuilt/<...> and out/host/<...> paths."""
   uname = os.uname()[0]
@@ -121,6 +136,7 @@ def FindToolchain():
   else:
     known_toolchains = []
 
+  logging.debug('FindToolcahin: known_toolchains=%s' % known_toolchains)
   # Look for addr2line to check for valid toolchain path.
   for (label, platform, target) in known_toolchains:
     toolchain_info = (label, platform, target);
@@ -206,6 +222,7 @@ def GetCandidates(dirs, filepart, candidate_fun):
   candidates = PathListJoin([out_dir], buildtype_list) + [CHROME_SYMBOLS_DIR]
   candidates = PathListJoin(candidates, dirs)
   candidates = PathListJoin(candidates, [filepart])
+  logging.debug('GetCandidates: prefiltered candidates = %s' % candidates)
   candidates = list(
       itertools.chain.from_iterable(map(candidate_fun, candidates)))
   candidates = sorted(candidates, key=os.path.getmtime, reverse=True)
@@ -237,7 +254,13 @@ def GetCrazyLib(apk_filename):
     if match:
       return match.group(1)
 
-def GetMatchingApks(device_apk_name):
+def GetApkFromLibrary(device_library_path):
+  match = re.match(r'.*/([^/]*)-[0-9]+(\/[^/]*)?\.apk$', device_library_path)
+  if not match:
+    return None
+  return match.group(1)
+
+def GetMatchingApks(package_name):
   """Find any APKs which match the package indicated by the device_apk_name.
 
   Args:
@@ -246,10 +269,6 @@ def GetMatchingApks(device_apk_name):
   Returns:
      A list of APK filenames which could contain the desired library.
   """
-  match = re.match('(.*)-[0-9]+[.]apk$', device_apk_name)
-  if not match:
-    return None
-  package_name = match.group(1)
   return filter(
       lambda candidate_apk:
           ApkMatchPackageName(GetAapt(), candidate_apk, package_name),
@@ -265,6 +284,7 @@ def MapDeviceApkToLibrary(device_apk_name):
     Name of the library which corresponds to that APK.
   """
   matching_apks = GetMatchingApks(device_apk_name)
+  logging.debug('MapDeviceApkToLibrary: matching_apks=%s' % matching_apks)
   for matching_apk in matching_apks:
     crazy_lib = GetCrazyLib(matching_apk)
     if crazy_lib:
@@ -280,18 +300,10 @@ def GetCandidateLibraries(library_name):
     A list of matching library filenames for library_name.
   """
   return GetCandidates(
-      ['lib', 'lib.target'], library_name,
+      ['lib', 'lib.target', '.'], library_name,
       lambda filename: filter(os.path.exists, [filename]))
 
 def TranslateLibPath(lib):
-  # SymbolInformation(lib, addr) receives lib as the path from symbols
-  # root to the symbols file. This needs to be translated to point to the
-  # correct .so path. If the user doesn't explicitly specify which directory to
-  # use, then use the most recently updated one in one of the known directories.
-  # If the .so is not found somewhere in CHROME_SYMBOLS_DIR, leave it
-  # untranslated in case it is an Android symbol in SYMBOLS_DIR.
-  library_name = os.path.basename(lib)
-
   # The filename in the stack trace maybe an APK name rather than a library
   # name. This happens when the library was loaded directly from inside the
   # APK. If this is the case we try to figure out the library name by looking
@@ -299,16 +311,30 @@ def TranslateLibPath(lib):
   # The name of the APK file on the device is of the form
   # <package_name>-<number>.apk. The APK file on the host may have any name
   # so we look at the APK badging to see if the package name matches.
-  if re.search('-[0-9]+[.]apk$', library_name):
-    mapping = MapDeviceApkToLibrary(library_name)
+  apk = GetApkFromLibrary(lib)
+  if apk is not None:
+    logging.debug('TranslateLibPath: apk=%s' % apk)
+    mapping = MapDeviceApkToLibrary(apk)
     if mapping:
-      library_name = mapping
+      lib = mapping
+
+  # SymbolInformation(lib, addr) receives lib as the path from symbols
+  # root to the symbols file. This needs to be translated to point to the
+  # correct .so path. If the user doesn't explicitly specify which directory to
+  # use, then use the most recently updated one in one of the known directories.
+  # If the .so is not found somewhere in CHROME_SYMBOLS_DIR, leave it
+  # untranslated in case it is an Android symbol in SYMBOLS_DIR.
+  library_name = os.path.basename(lib)
+
+  logging.debug('TranslateLibPath: lib=%s library_name=%s' % (lib, library_name))
 
   candidate_libraries = GetCandidateLibraries(library_name)
+  logging.debug('TranslateLibPath: candidate_libraries=%s' % candidate_libraries)
   if not candidate_libraries:
     return lib
 
   library_path = os.path.relpath(candidate_libraries[0], SYMBOLS_DIR)
+  logging.debug('TranslateLibPath: library_path=%s' % library_path)
   return '/' + library_path
 
 def SymbolInformation(lib, addr, get_detailed_info):