stack.py: Remove relocation_packer flags & improve logging

* Changes all status messages from print() -> logging.info()
* Adds a --quiet flag
* Adds a few more lines to output that script was dropping:
  Tombstone written to:|Abort message:|Revision:|Build fingerprint:

Bug: 917452
Change-Id: Id6f572aee3d6ebe9ceb5ce035b032862d24e2ae6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1978904
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Richard Coles <torne@chromium.org>
Cr-Commit-Position: refs/heads/master@{#727176}

third_party/android_platform/development/scripts/stack.py

@@ -23,7 +23,6 @@ import os
 import sys
 
 import stack_core
-import stack_libs
 import subprocess
 import symbol
 import sys
@@ -64,12 +63,6 @@ def PrintUsage():
   print "       Overrides the default apks directory. Useful if a bundle APKS"
   print "       file has been unzipped into a temporary directory."
   print
-  print "  --packed-relocation-adjustments"
-  print "  --no-packed-relocation-adjustments"
-  print "       turn packed relocation adjustment on and off (default is off)"
-  print "       If running on pre-M Android and the stack trace appears to"
-  print "       make no sense, try turning this feature on."
-  print
   print "  --symbols-zip=path"
   print "       the path to a symbols zip file, such as"
   print "       =dream-symbols-12345.zip"
@@ -90,6 +83,9 @@ def PrintUsage():
   print "       shared lib which is loaded from APK, this doesn't work for"
   print "       component build."
   print
+  print "  --quiet"
+  print "       Show less logging"
+  print
   print "  --verbose"
   print "       enable extra logging, particularly for debugging failed"
   print "       symbolization"
@@ -122,7 +118,7 @@ def UnzipSymbols(symbolfile, symdir=None):
   if not os.path.exists(symdir):
     os.makedirs(symdir)
 
-  print "extracting %s..." % symbolfile
+  logging.info('extracting %s...', symbolfile)
   saveddir = os.getcwd()
   os.chdir(symdir)
   try:
@@ -147,10 +143,9 @@ def UnzipSymbols(symbolfile, symdir=None):
 def main(argv, test_symbolizer=None):
   try:
     options, arguments = getopt.getopt(argv, "", [
-        "packed-relocation-adjustments", "no-packed-relocation-adjustments",
         "more-info", "less-info", "chrome-symbols-dir=", "output-directory=",
-        "apks-directory=", "symbols-dir=", "symbols-zip=", "packed-lib=",
-        "arch=", "fallback-monochrome", "verbose", "help"
+        "apks-directory=", "symbols-dir=", "symbols-zip=", "arch=",
+        "fallback-monochrome", "verbose", "quiet", "help",
     ])
   except getopt.GetoptError, _:
     PrintUsage()
@@ -159,8 +154,8 @@ def main(argv, test_symbolizer=None):
   more_info = False
   fallback_monochrome = False
   arch_defined = False
-  packed_libs = []
   apks_directory = None
+  log_level = logging.INFO
   for option, value in options:
     if option == "--help":
       PrintUsage()
@@ -178,8 +173,6 @@ def main(argv, test_symbolizer=None):
       constants.SetOutputDirectory(os.path.abspath(value))
     elif option == "--apks-directory":
       apks_directory = os.path.abspath(value)
-    elif option == "--packed-lib":
-      packed_libs.append(os.path.abspath(os.path.expanduser(value)))
     elif option == "--more-info":
       more_info = True
     elif option == "--less-info":
@@ -187,59 +180,50 @@ def main(argv, test_symbolizer=None):
     elif option == "--fallback-monochrome":
       fallback_monochrome = True
     elif option == "--verbose":
-      logging.basicConfig(level=logging.DEBUG)
-    elif option in (
-        '--packed-relocation-adjustments',
-        '--no-packed-relocation-adjustments'):
-      print ('--[no-]packed-relocation-adjustments options are deprecated. '
-             'Specify packed libs directory instead.')
+      log_level = logging.DEBUG
+    elif option == "--quiet":
+      log_level = logging.WARNING
 
   if len(arguments) > 1:
     PrintUsage()
 
+  logging.basicConfig(level=log_level)
   # Do an up-front test that the output directory is known.
   if not symbol.CHROME_SYMBOLS_DIR:
     constants.CheckOutputDirectory()
 
-  print ("Reading Android symbols from: "
-         + os.path.normpath(symbol.SYMBOLS_DIR))
+  logging.info('Reading Android symbols from: %s',
+               os.path.normpath(symbol.SYMBOLS_DIR))
   chrome_search_path = symbol.GetLibrarySearchPaths()
-  print ("Searching for Chrome symbols from within: "
-         + ':'.join((os.path.normpath(d) for d in chrome_search_path)))
+  logging.info('Searching for Chrome symbols from within: %s',
+               ':'.join((os.path.normpath(d) for d in chrome_search_path)))
 
   rootdir = None
   if zip_arg:
     rootdir, symbol.SYMBOLS_DIR = UnzipSymbols(zip_arg)
 
   if not arguments or arguments[0] == "-":
-    print "Reading native crash info from stdin"
+    logging.info('Reading native crash info from stdin')
     with llvm_symbolizer.LLVMSymbolizer() as symbolizer:
       stack_core.StreamingConvertTrace(sys.stdin, {}, more_info,
                                        fallback_monochrome, arch_defined,
                                        symbolizer, apks_directory)
   else:
-    print "Searching for native crashes in: " + os.path.realpath(arguments[0])
+    logging.info('Searching for native crashes in: %s',
+                 os.path.realpath(arguments[0]))
     f = open(arguments[0], "r")
 
     lines = f.readlines()
     f.close()
 
-    version = stack_libs.GetTargetAndroidVersionNumber(lines)
-    if version is None:
-      print ("Unknown Android release, "
-             "consider passing --packed-lib.")
-    elif version < _ANDROID_M_MAJOR_VERSION and not packed_libs:
-      print ("Pre-M Android release detected, "
-             "but --packed-lib not specified. Stack symbolization may fail.")
-
-    if (version is None or version < _ANDROID_M_MAJOR_VERSION) and packed_libs:
-      load_vaddrs = stack_libs.GetLoadVaddrs(stripped_libs=packed_libs)
-    else:
+    # This used to be required when ELF logical addresses did not align with
+    # physical addresses, which happened when relocations were converted to APS2
+    # format post-link via relocation_packer tool.
     load_vaddrs = {}
 
     with llvm_symbolizer.LLVMSymbolizer() as symbolizer:
-      print ("Searching for Chrome symbols from within: "
-             + ':'.join((os.path.normpath(d) for d in chrome_search_path)))
+      logging.info('Searching for Chrome symbols from within: %s',
+                   ':'.join((os.path.normpath(d) for d in chrome_search_path)))
       stack_core.ConvertTrace(lines, load_vaddrs, more_info,
                               fallback_monochrome, arch_defined,
                               test_symbolizer or symbolizer, apks_directory)
@@ -247,7 +231,7 @@ def main(argv, test_symbolizer=None):
   if rootdir:
     # be a good citizen and clean up...os.rmdir and os.removedirs() don't work
     cmd = "rm -rf \"%s\"" % rootdir
-    print "\ncleaning up (%s)" % cmd
+    logging.info('cleaning up (%s)', cmd)
     os.system(cmd)
 
 if __name__ == "__main__":

third_party/android_platform/development/scripts/stack.pydeps

@@ -17,5 +17,4 @@
 ../../../catapult/devil/devil/constants/exit_codes.py
 stack.py
 stack_core.py
-stack_libs.py
 symbol.py

third_party/android_platform/development/scripts/stack_core.py

@@ -52,6 +52,8 @@ _THREAD_LINE = re.compile('(.*)(\-\-\- ){15}\-\-\-')
 _DALVIK_JNI_THREAD_LINE = re.compile("(\".*\" prio=[0-9]+ tid=[0-9]+ NATIVE.*)")
 _DALVIK_NATIVE_THREAD_LINE = re.compile("(\".*\" sysTid=[0-9]+ nice=[0-9]+.*)")
 _JAVA_STDERR_LINE = re.compile("([0-9]+)\s+[0-9]+\s+.\s+System.err:\s*(.+)")
+_MISC_HEADER = re.compile(
+    '(?:Tombstone written to:|Abort message:|Revision:|Build fingerprint:).*')
 
 # Matches LOG(FATAL) lines, like the following example:
 #   [FATAL:source_file.cc(33)] Check failed: !instances_.empty()
@@ -172,14 +174,14 @@ def StreamingConvertTrace(_, load_vaddrs, more_info, fallback_monochrome,
   so_dirs = []
   in_stack = False
   def ConvertStreamingChunk():
-    print "Stack found. Symbolizing..."
+    logging.info("Stack found. Symbolizing...")
     if so_dirs:
       UpdateLibrarySearchPath(so_dirs)
     # if arch isn't defined in command line, find it from log
     if not arch_defined:
       arch = _FindAbi(useful_lines)
       if arch:
-        print ('Find ABI:' + arch)
+        print 'Symbolizing stack using ABI=' + arch
         symbol.ARCH = arch
     ResolveCrashSymbol(list(useful_lines), more_info, llvm_symbolizer)
 
@@ -214,7 +216,8 @@ def ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome,
 
   chunks = [lines[i: i+_CHUNK_SIZE] for i in xrange(0, len(lines), _CHUNK_SIZE)]
 
-  use_multiprocessing = os.environ.get('STACK_DISABLE_ASYNC') != '1'
+  use_multiprocessing = len(chunks) > 1 and (
+      os.environ.get('STACK_DISABLE_ASYNC') != '1')
   if use_multiprocessing:
     pool = multiprocessing.Pool(processes=_DEFAULT_JOBS)
     results = pool.map(PreProcessLog(load_vaddrs, apks_directory), chunks)
@@ -240,7 +243,7 @@ def ConvertTrace(lines, load_vaddrs, more_info, fallback_monochrome,
   if not arch_defined:
     arch = _FindAbi(useful_log)
     if arch:
-      print ('Find ABI:' + arch)
+      print 'Symbolizing stack using ABI:', arch
       symbol.ARCH = arch
 
   ResolveCrashSymbol(list(useful_log), more_info, llvm_symbolizer)
@@ -286,8 +289,8 @@ class PreProcessLog:
         # we can update library search path in main process.
         if not os.path.samefile(constants.GetOutDirectory(), so_dir):
           self._so_dirs.append(so_dir)
-        print ("Detected: %s is %s which is loaded directly from APK."
-                % (host_so, soname))
+        logging.info('Detected: %s is %s which is loaded directly from APK.',
+                     host_so, soname)
     return soname
 
   def _AdjustAddress(self, address, lib):
@@ -327,7 +330,8 @@ class PreProcessLog:
           or _LOG_FATAL_LINE.search(line)
           or _DEBUG_TRACE_LINE.search(line)
           or _ABI_LINE.search(line)
-          or _JAVA_STDERR_LINE.search(line)):
+          or _JAVA_STDERR_LINE.search(line)
+          or _MISC_HEADER.search(line)):
         useful_log.append(line)
         continue
 
@@ -396,9 +400,10 @@ def ResolveCrashSymbol(lines, more_info, llvm_symbolizer):
     dalvik_jni_thread_header = _DALVIK_JNI_THREAD_LINE.search(line)
     dalvik_native_thread_header = _DALVIK_NATIVE_THREAD_LINE.search(line)
     log_fatal_header = _LOG_FATAL_LINE.search(line)
+    misc_header = _MISC_HEADER.search(line)
     if (process_header or signal_header or register_header or thread_header or
         dalvik_jni_thread_header or dalvik_native_thread_header or
-        log_fatal_header) :
+        log_fatal_header or misc_header):
       if trace_lines or value_lines:
         java_lines = []
         if pid != -1 and pid in java_stderr_by_pid:
@@ -425,6 +430,8 @@ def ResolveCrashSymbol(lines, more_info, llvm_symbolizer):
         print dalvik_native_thread_header.group(1)
       if log_fatal_header:
         print log_fatal_header.group(1)
+      if misc_header:
+        print misc_header.group(0)
       continue
 
     match = _TRACE_LINE.match(line) or _DEBUG_TRACE_LINE.match(line)
@@ -499,7 +506,7 @@ def UpdateLibrarySearchPath(so_dirs):
       raise Exception("Found different so dirs, they are %s", repr(so_dir))
     else:
       search_path = so_dir.pop()
-      print "Search libraries in " + search_path
+      logging.info("Search libraries in %s", search_path)
       symbol.SetSecondaryAbiOutputPath(search_path)
 
 
@@ -614,7 +621,7 @@ def _FindSharedLibraryFromAPKs(output_directory, apks_directory, offset):
   if number_of_library == 1:
     return shared_libraries[0]
   elif number_of_library > 1:
-    print "More than one libraries could be loaded from APK."
+    logging.warning("More than one libraries could be loaded from APK.")
   return (None, None)
 
 

third_party/android_platform/development/scripts/stack_libs.py

-#!/usr/bin/env python
-#
-# Copyright (c) 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Identifies address adjustments required for native crash dumps."""
-
-import glob
-import os.path
-import re
-import subprocess
-
-
-_BASE_APK = 'base.apk'
-_LIBCHROME_SO = 'libchrome.so'
-
-_BUILD_FINGERPRINT_RE = re.compile('.*Build fingerprint: (.*)$')
-
-
-def GetTargetAndroidVersionNumber(lines):
-  """Return the Android major version number from the build fingerprint.
-
-  Args:
-    lines: Lines read from the tombstone file, before preprocessing.
-  Returns:
-    5, 6, etc, or None if not determinable (developer build?)
-  """
-  # For example, "Build fingerprint: 'Android/aosp_flo/flo:5.1.1/...'" is 5.
-  for line in lines:
-    m = _BUILD_FINGERPRINT_RE.match(line)
-    if not m:
-      continue
-    fingerprint = m.group(1)
-    try:
-      version = fingerprint.split('/')[2].split(':')[1].split('.')[0]
-      return int(version)
-    except Exception:
-      pass
-  return None
-
-
-def _HasElfHeader(path):
-  """Return True if the file at the given path has an ELF magic header.
-
-  Minimal check only, for 'ELF' in bytes 1 to 3 of the file. Filters out
-  the zero-byte false-positives such as libchromeview.so returned by glob.
-
-  Args:
-    path: Path to file to check.
-  Returns:
-    True or False
-  """
-  with open(path) as stream:
-    elf_header = stream.read(4)
-    return len(elf_header) == 4 and elf_header[1:4] == 'ELF'
-
-
-def _ReadElfProgramHeaders(lib):
-  """Return an iterable of program headers, from 'readelf -l ...'.
-
-  Uses the platform readelf in all cases. This is somewhat lazy, but suffices
-  in practice because program headers in ELF files are architecture-agnostic.
-
-  Args:
-    lib: Library file to read.
-  Returns:
-    [readelf -l output line, ...]
-  """
-  string = subprocess.check_output(['readelf', '-l', lib])
-  return string.split('\n')
-
-
-def _FindMinLoadVaddr(lib):
-  """Return the minimum VirtAddr field of all library LOAD segments.
-
-  Args:
-    lib: Library file to read.
-  Returns:
-    Min VirtAddr field for all LOAD segments, or 0 if none found.
-  """
-  vaddrs = []
-  # Locate LOAD lines and find the smallest VirtAddr field, eg:
-  #   Type       Offset    VirtAddr   PhysAddr   FileSiz   MemSiz    Flg Align
-  #   LOAD       0x000000  0x001d6000 0x001d6000 0x20f63fc 0x20f63fc R E 0x1000
-  #   LOAD       0x20f6970 0x022cd970 0x022cd970 0x182df8  0x1b4490  RW  0x1000
-  # would return 0x1d6000. Ignores all non-LOAD lines.
-  for line in _ReadElfProgramHeaders(lib):
-    elements = line.split()
-    if elements and elements[0] == 'LOAD':
-      vaddrs.append(int(elements[2], 16))
-  if vaddrs:
-    return min(vaddrs)
-  return 0
-
-
-def GetLoadVaddrs(stripped_libs=None, stripped_libs_dir=None):
-  """Return a dict of minimum VirtAddr for libraries in the given directory.
-
-  The dictionary returned may be passed to stack_core.ConvertTrace(). In
-  pre-M Android releases the addresses printed by debuggerd into tombstones
-  do not take account of non-zero vaddrs. Here we collect this information,
-  so that we can use it later to correct such debuggerd tombstones.
-
-  Args:
-    stripped_libs_dir: Path to directory containing apk's stripped libraries.
-  Returns:
-    {'libchrome.so': 12345, ...}
-  """
-  if not stripped_libs:
-    stripped_libs = []
-  if stripped_libs_dir:
-    stripped_libs.extend(glob.glob(os.path.join(stripped_libs_dir, '*.so')))
-  libs = [l for l in stripped_libs if _HasElfHeader(l)]
-
-  load_vaddrs = {}
-  for lib in libs:
-    min_vaddr = _FindMinLoadVaddr(lib)
-    if min_vaddr:
-      # Store with the library basename as the key. This is because once on
-      # the device its path may not fully match its place in the APK staging
-      # directory
-      load_vaddrs[os.path.basename(lib)] = min_vaddr
-
-  # Direct load from APK causes debuggerd to tag trace lines as if from the
-  # file .../base.apk. So if we encounter a libchrome.so with packed
-  # relocations, replicate this as base.apk so that later adjustment code
-  # finds the appropriate adjustment.
-  if _LIBCHROME_SO in load_vaddrs:
-    load_vaddrs[_BASE_APK] = load_vaddrs[_LIBCHROME_SO]
-
-  return load_vaddrs