[Supersize] Add LTO detection using LLD .map files.

LLD-LTO flow (TODO) will be quite different from LLD flow. Supersize
will need to tell them apart (and store them using distinct linker names
in the .size file). This CL adds the functionality to detect LTO usage
from LLD .map files.

Previously linker detection (to discern Gold, and 2 versions of LLD
files) only needed to examine the first line of .map file. But for
LTO (LLD only), we'll need to look beyond this. The signal we chose is
to look for 'thinlto-cache' prefix at locations where paths usually
reside in .map files. To optimize for speed, we choose and scan one
'indicator section' (.ARM.exidx or .reloc, but easy to tweak).

Details:
- Refactor: Previously linker_map_parser.DetectLinkerNameFromMapFile()
  was called twice: to computie metadata, and in MapFileParser. Since
  the function is becoming more complex, we new call it once (for
  metadata), then pipe the information to MapFileParser to use.
- Add linker_map_parser._DetectLto() to handle most of the parsing. As
  optimization, we use ad-hoc string processing instead of Regex.
- Minor cleanup on version 0 vs. 1 difference for LLD.

After this change, the list of linker names are:
  {'gold', 'lld_v0', 'lld-lto_v0', 'lld_v1', 'lld-lto_v1'}.

Bug: 723798
Change-Id: I0498e50446be32a0335bb4781d60a6b72d7e87b5
Reviewed-on: https://chromium-review.googlesource.com/1107037
Commit-Queue: Samuel Huang <huangs@chromium.org>
Reviewed-by: Samuel Huang <huangs@chromium.org>
Reviewed-by: agrieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#568723}

tools/binary_size/libsupersize/archive.py

@@ -693,7 +693,7 @@ def CreateMetadata(map_path, elf_path, apk_path, tool_prefix, output_directory,
     apk_path: Path to the .apk file to measure.
     tool_prefix: Prefix for c++filt & nm.
     output_directory: Build output directory.
-    linker_name: "gold", "lld", or None
+    linker_name: 'gold', 'lld_v#' (# is a number), 'lld-lto_v#', or None.
 
   Returns:
     None if |elf_path| is not supplied. Otherwise returns dict mapping string
@@ -747,7 +747,7 @@ def _ResolveThinArchivePaths(raw_symbols, thin_archives):
 
 
 def _ParseElfInfo(map_path, elf_path, tool_prefix, track_string_literals,
-                  outdir_context=None):
+                  outdir_context=None, linker_name=None):
   """Adds ELF section sizes and symbols."""
   if elf_path:
     # Run nm on the elf file to retrieve the list of symbol names per-address.
@@ -773,7 +773,7 @@ def _ParseElfInfo(map_path, elf_path, tool_prefix, track_string_literals,
   logging.info('Parsing Linker Map')
   with _OpenMaybeGz(map_path) as map_file:
     section_sizes, raw_symbols = (
-        linker_map_parser.MapFileParser().Parse(map_file))
+        linker_map_parser.MapFileParser().Parse(linker_name, map_file))
 
     if outdir_context and outdir_context.thin_archives:
       _ResolveThinArchivePaths(raw_symbols, outdir_context.thin_archives)
@@ -1125,7 +1125,7 @@ def _CalculateElfOverhead(section_sizes, elf_path):
 def CreateSectionSizesAndSymbols(
       map_path=None, tool_prefix=None, output_directory=None, elf_path=None,
       apk_path=None, track_string_literals=True, metadata=None,
-      apk_so_path=None, pak_files=None, pak_info_file=None,
+      apk_so_path=None, pak_files=None, pak_info_file=None, linker_name=None,
       knobs=SectionSizeKnobs()):
   """Creates sections sizes and symbols for a SizeInfo.
 
@@ -1184,7 +1184,8 @@ def CreateSectionSizesAndSymbols(
         thin_archives=thin_archives)
 
   section_sizes, raw_symbols = _ParseElfInfo(
-      map_path, elf_path, tool_prefix, track_string_literals, outdir_context)
+      map_path, elf_path, tool_prefix, track_string_literals,
+      outdir_context=outdir_context, linker_name=linker_name)
   elf_overhead_size = _CalculateElfOverhead(section_sizes, elf_path)
 
   pak_symbols_by_id = None
@@ -1318,7 +1319,7 @@ def _ParseGnArgs(args_path):
 
 def _DetectLinkerName(map_path):
   with _OpenMaybeGz(map_path) as map_file:
-    return linker_map_parser.DetectLinkerNameFromMapFileHeader(next(map_file))
+    return linker_map_parser.DetectLinkerNameFromMapFile(map_file)
 
 
 def _ElfInfoFromApk(apk_path, apk_so_path, tool_prefix):
@@ -1439,6 +1440,7 @@ def DeduceMainPaths(args, parser):
                    'linker map file.')
 
   linker_name = _DetectLinkerName(map_path)
+  logging.info('Linker name: %s' % linker_name)
   tool_prefix_finder = path_util.ToolPrefixFinder(
       value=args.tool_prefix,
       output_directory_finder=output_directory_finder,
@@ -1470,7 +1472,8 @@ def Run(args, parser):
       apk_path=apk_path, output_directory=output_directory,
       track_string_literals=args.track_string_literals,
       metadata=metadata, apk_so_path=apk_so_path,
-      pak_files=args.pak_file, pak_info_file=args.pak_info_file, knobs=knobs)
+      pak_files=args.pak_file, pak_info_file=args.pak_info_file,
+      linker_name=linker_name, knobs=knobs)
   size_info = CreateSizeInfo(
       section_sizes, raw_symbols, metadata=metadata, normalize_names=False)
 

tools/binary_size/libsupersize/integration_test.py

@@ -183,16 +183,18 @@ class IntegrationTest(unittest.TestCase):
         pak_files = [_TEST_APK_PAK_PATH]
         pak_info_file = _TEST_PAK_INFO_PATH
       metadata = None
+      linker_name = 'gold'
       if use_elf:
         with _AddMocksToPath():
           metadata = archive.CreateMetadata(
               _TEST_MAP_PATH, elf_path, apk_path, _TEST_TOOL_PREFIX,
-              output_directory, 'gold')
+              output_directory, linker_name)
       section_sizes, raw_symbols = archive.CreateSectionSizesAndSymbols(
           map_path=_TEST_MAP_PATH, tool_prefix=_TEST_TOOL_PREFIX,
           elf_path=elf_path, output_directory=output_directory,
           apk_path=apk_path, apk_so_path=apk_so_path, metadata=metadata,
-          pak_files=pak_files, pak_info_file=pak_info_file, knobs=knobs)
+          pak_files=pak_files, pak_info_file=pak_info_file,
+          linker_name=linker_name, knobs=knobs)
       IntegrationTest.cached_size_info[cache_key] = archive.CreateSizeInfo(
           section_sizes, raw_symbols, metadata=metadata)
     return copy.deepcopy(IntegrationTest.cached_size_info[cache_key])

tools/binary_size/libsupersize/linker_map_parser.py

@@ -287,6 +287,7 @@ class MapFileParserLld(object):
   _LINE_RE_V0 = re.compile(r'([0-9a-f]+)\s+([0-9a-f]+)\s+(\d+) ( *)(.*)')
   _LINE_RE_V1 = re.compile(
       r'\s*[0-9a-f]+\s+([0-9a-f]+)\s+([0-9a-f]+)\s+(\d+) ( *)(.*)')
+  _LINE_RE = [_LINE_RE_V0, _LINE_RE_V1]
 
   def __init__(self, linker_name):
     self._linker_name = linker_name
@@ -325,13 +326,13 @@ class MapFileParserLld(object):
 # 00000000002010c0 0000000000000000     0                 frame_dummy
 # 00000000002010ed 0000000000000071     1         a.o:(.text)
 # 00000000002010ed 0000000000000071     0                 main
+    # Extract e.g., 'lld_v0' -> 0, or 'lld-lto_v1' -> 1.
+    map_file_version = int(self._linker_name.split('_v')[1])
+    pattern = MapFileParserLld._LINE_RE[map_file_version]
+
     sym_maker = _SymbolMaker()
     cur_section = None
     cur_section_is_useful = None
-    if self._linker_name.endswith('v1'):
-      pattern = self._LINE_RE_V1
-    else:
-      pattern = self._LINE_RE_V0
 
     for line in lines:
       m = pattern.match(line)
@@ -391,19 +392,58 @@ class MapFileParserLld(object):
     return self._section_sizes, sym_maker.syms
 
 
-def DetectLinkerNameFromMapFileHeader(first_line):
+def _DetectLto(lines):
+  """Scans LLD linker map file and returns whether LTO was used."""
+  # It's assumed that the first line in |lines| was consumed to determine that
+  # LLD was used. Seek 'thinlto-cache' prefix within an "indicator section" as
+  # indicator for LTO.
+  found_indicator_section = False
+  # Potential names of "main section". Only one gets used.
+  indicator_section_set = set(['.rodata', '.ARM.exidx'])
+  start_pos = -1
+  for line in lines:
+    # Shortcut to avoid regex: The first line seen (second line in file) should
+    # start a section, and start with '.', e.g.:
+    #     194      194       13     1 .interp
+    # Assign |start_pos| as position of '.', and trim everything before!
+    if start_pos < 0:
+      start_pos = line.index('.')
+    if len(line) < start_pos:
+      continue
+    line = line[start_pos:]
+    tok = line.lstrip()  # Allow whitespace at right.
+    indent_size = len(line) - len(tok)
+    if indent_size == 0:  # Section change.
+      if found_indicator_section:  # Exit if just visited "main section".
+        break
+      if tok.strip() in indicator_section_set:
+        found_indicator_section = True
+    elif indent_size == 8:
+      if found_indicator_section:
+        if tok.startswith('thinlto-cache'):
+          return True
+  return False
+
+
+def DetectLinkerNameFromMapFile(lines):
+  """Scans linker map file, and returns a coded linker name."""
+  first_line = next(lines)
+
   if first_line.startswith('Address'):
-    return 'lld_v0'
-  elif first_line.lstrip().startswith('VMA'):
-    return 'lld_v1'
+    return 'lld-lto_v0' if _DetectLto(lines) else 'lld_v0'
+
+  if first_line.lstrip().startswith('VMA'):
+    return 'lld-lto_v1' if _DetectLto(lines) else 'lld_v1'
+
   if first_line.startswith('Archive member'):
     return 'gold'
+
   raise Exception('Invalid map file: ' + first_line)
 
 
 class MapFileParser(object):
   """Parses a linker map file, with heuristic linker detection."""
-  def Parse(self, lines):
+  def Parse(self, linker_name, lines):
     """Parses a linker map file.
 
     Args:
@@ -412,7 +452,7 @@ class MapFileParser(object):
     Returns:
       A tuple of (section_sizes, symbols).
     """
-    linker_name = DetectLinkerNameFromMapFileHeader(next(lines))
+    next(lines)  # Consume the first line of headers.
     if linker_name.startswith('lld'):
       inner_parser = MapFileParserLld(linker_name)
     elif linker_name == 'gold':