SuperSize: Truncate long tail of symbols in diff output

This should greatly reduce the amount of output generated when one alias
is added/removed from a large alias group.

It also simplifies Sorted(), so that it really just sorts by pss and
doesn't put bss & padding-only symbols at the bottom. I think this is
fine for now, and if we still find that these symbols are confusing
people, we could change the diff command to explicitly print bss &
padding symbols after the main list of them (via multiple Print()
statements)

Change-Id: I23db60de6cffa6bf8a9399cb30e46cf5d92ad0f6
Reviewed-on: https://chromium-review.googlesource.com/1176167
Commit-Queue: agrieve <agrieve@chromium.org>
Reviewed-by: Samuel Huang <huangs@chromium.org>
Cr-Commit-Position: refs/heads/master@{#583491}

tools/binary_size/libsupersize/canned_queries.py

@@ -36,7 +36,7 @@ class _Grouper(object):
       self.groups.extend(others_by_path)
 
     logging.debug('Finalized')
-    return models.SymbolGroup(self.groups, is_sorted=True)
+    return models.SymbolGroup(self.groups)
 
 
 def _CategorizeByChromeComponent(symbols):

tools/binary_size/libsupersize/describe.py

@@ -278,7 +278,28 @@ class DescriberText(Describer):
     indent_prefix = '> ' * indent
     diff_prefix = ''
     total = group.pss
+    # is_default_sorted ==> sorted by abs(PSS) from largest to smallest.
+    if group.is_default_sorted:
+      # Skip long tail of small symbols (useful for diffs where aliases change).
+      # Long tail is defined as:
+      #   * Accounts for < .5% of PSS
+      #   * Symbols are smaller than 1.0 byte (by PSS)
+      #   * Always show at least 50 symbols.
+      min_remaining_pss_to_show = max(1024, total / 1000 * 5)
+      min_symbol_pss_to_show = 1.0
+      min_symbols_to_show = 50
+
     for index, s in enumerate(group):
+      if group.is_default_sorted and not self.verbose:
+        remaining_pss = total - running_total
+        if (index >= min_symbols_to_show and
+            abs(remaining_pss) < min_remaining_pss_to_show and
+            abs(s.pss) < min_symbol_pss_to_show):
+          remaining_count = len(group) - index
+          yield '{}Skipping {} tiny symbols comprising {} bytes.'.format(
+              indent_prefix, remaining_count, _FormatPss(remaining_pss))
+          break
+
       if group.IsBss() or not s.IsBss():
         running_total += s.pss
         running_percent = _Divide(running_total, total)

tools/binary_size/libsupersize/models.py

@@ -569,12 +569,13 @@ class SymbolGroup(BaseSymbol):
       'template_name',
       'name',
       'section_name',
-      'is_sorted',
+      'is_default_sorted',  # True for groups created by Sorted()
   )
 
   # template_name and full_name are useful when clustering symbol clones.
   def __init__(self, symbols, filtered_symbols=None, full_name=None,
-               template_name=None, name='', section_name=None, is_sorted=False):
+               template_name=None, name='', section_name=None,
+               is_default_sorted=False):
     self._padding = None
     self._size = None
     self._pss = None
@@ -584,7 +585,7 @@ class SymbolGroup(BaseSymbol):
     self.template_name = template_name if template_name is not None else name
     self.name = name or ''
     self.section_name = section_name or SECTION_MULTIPLE
-    self.is_sorted = is_sorted
+    self.is_default_sorted = is_default_sorted
 
   def __repr__(self):
     return 'Group(full_name=%s,count=%d,size=%d)' % (
@@ -624,7 +625,7 @@ class SymbolGroup(BaseSymbol):
   def __add__(self, other):
     self_ids = set(id(s) for s in self)
     after_symbols = self._symbols + [s for s in other if id(s) not in self_ids]
-    return self._CreateTransformed(after_symbols, is_sorted=False)
+    return self._CreateTransformed(after_symbols, is_default_sorted=False)
 
   def index(self, item):
     return self._symbols.index(item)
@@ -712,29 +713,28 @@ class SymbolGroup(BaseSymbol):
 
   def _CreateTransformed(self, symbols, filtered_symbols=None, full_name=None,
                          template_name=None, name=None, section_name=None,
-                         is_sorted=None):
-    if is_sorted is None:
-      is_sorted = self.is_sorted
+                         is_default_sorted=None):
+    if is_default_sorted is None:
+      is_default_sorted = self.is_default_sorted
     if section_name is None:
       section_name = self.section_name
     return self.__class__(symbols, filtered_symbols=filtered_symbols,
                           full_name=full_name, template_name=template_name,
                           name=name, section_name=section_name,
-                          is_sorted=is_sorted)
+                          is_default_sorted=is_default_sorted)
 
   def Sorted(self, cmp_func=None, key=None, reverse=False):
+    """Sorts by abs(PSS)."""
+    is_default_sorted = False
     if cmp_func is None and key is None:
-      if self.IsDelta():
-        key = lambda s: (s.diff_status == DIFF_STATUS_UNCHANGED, s.IsBss(),
-                         s.size_without_padding == 0, -abs(s.pss), s.name)
-      else:
-        key = lambda s: (
-            s.IsBss(), s.size_without_padding == 0, -abs(s.pss), s.name)
+      is_default_sorted = not reverse
+      # Sort by PSS, but ensure ties are broken in a consistent manner.
+      key = lambda s: (-abs(s.pss), s.full_name, s.object_path, s.section_name)
 
     after_symbols = sorted(self._symbols, cmp_func, key, reverse)
     return self._CreateTransformed(
         after_symbols, filtered_symbols=self._filtered_symbols,
-        is_sorted=True)
+        is_default_sorted=is_default_sorted)
 
   def SortedByName(self, reverse=False):
     return self.Sorted(key=(lambda s:s.name), reverse=reverse)
@@ -873,7 +873,7 @@ class SymbolGroup(BaseSymbol):
     """
     return self._CreateTransformed(
         self._filtered_symbols, filtered_symbols=self._symbols,
-        section_name=SECTION_MULTIPLE, is_sorted=False)
+        section_name=SECTION_MULTIPLE, is_default_sorted=False)
 
   def GroupedBy(self, func, min_count=0, group_factory=None):
     """Returns a SymbolGroup of SymbolGroups, indexed by |func|.

tools/binary_size/libsupersize/testdata/Console.golden

@@ -4,7 +4,7 @@ Entering interactive Python shell. Quick reference:
 SizeInfo: metadata, native_symbols, pak_symbols, raw_symbols, section_sizes, size_path, symbols
 Symbol: FlagsString, IsBss, IsDelta, IsDex, IsGeneratedByToolchain, IsGroup, IsNative, IsOther, IsOverhead, IsPak, IsStringLiteral, IterLeafSymbols, address, aliases, component, end_address, flags, full_name, generated_source, is_anonymous, name, num_aliases, object_path, padding, padding_pss, pss, pss_without_padding, section, section_name, size, size_without_padding, source_path, template_name
 
-SymbolGroup (extends Symbol): CountUniqueSymbols, Filter, GroupedBy, GroupedByAliases, GroupedByComponent, GroupedByFullName, GroupedByName, GroupedByPath, GroupedBySectionName, Inverted, IterUniqueSymbols, SetName, Sorted, SortedByAddress, SortedByCount, SortedByName, WhereAddressInRange, WhereComponentMatches, WhereFullNameMatches, WhereGeneratedByToolchain, WhereHasAnyAttribution, WhereHasComponent, WhereHasPath, WhereInSection, WhereIsDex, WhereIsGroup, WhereIsNative, WhereIsPak, WhereIsTemplate, WhereMatches, WhereNameMatches, WhereObjectPathMatches, WherePathMatches, WherePssBiggerThan, WhereSizeBiggerThan, WhereSourceIsGenerated, WhereSourcePathMatches, WhereTemplateNameMatches, index, is_sorted
+SymbolGroup (extends Symbol): CountUniqueSymbols, Filter, GroupedBy, GroupedByAliases, GroupedByComponent, GroupedByFullName, GroupedByName, GroupedByPath, GroupedBySectionName, Inverted, IterUniqueSymbols, SetName, Sorted, SortedByAddress, SortedByCount, SortedByName, WhereAddressInRange, WhereComponentMatches, WhereFullNameMatches, WhereGeneratedByToolchain, WhereHasAnyAttribution, WhereHasComponent, WhereHasPath, WhereInSection, WhereIsDex, WhereIsGroup, WhereIsNative, WhereIsPak, WhereIsTemplate, WhereMatches, WhereNameMatches, WhereObjectPathMatches, WherePathMatches, WherePssBiggerThan, WhereSizeBiggerThan, WhereSourceIsGenerated, WhereSourcePathMatches, WhereTemplateNameMatches, index, is_default_sorted
 
 DeltaSizeInfo: after, before, native_symbols, pak_symbols, raw_symbols, section_sizes, symbols
 DeltaSymbol (extends Symbol): after_symbol, before_symbol, diff_status

tools/binary_size/libsupersize/testdata/Diff_Basic.golden

@@ -64,14 +64,14 @@ Index | Running Total | Section@Address | ...
                source_path= 	object_path=
                flags={}  name=IDS_WEB_FONT_FAMILY
                     full_name=../../ui/strings/app_locale_settings.grd: IDS_WEB_FONT_FAMILY
-- 2)       -112 (100.0%) p@0x0        -9 (9->0)  num_aliases=1
-               source_path= 	object_path=
-               flags={}  name=IDS_WEB_FONT_SIZE
-                    full_name=../../ui/strings/app_locale_settings.grd: IDS_WEB_FONT_SIZE
-~ 3)       -130 (116.1%) p@0x0        -18 (0->0)  num_aliases=1
+~ 2)       -121 (108.0%) p@0x0        -18 (0->0)  num_aliases=1
                source_path= 	object_path=
                flags={}  name=../../../mock_apk/assets/en-US.pak
                     full_name=foo: ../../../mock_apk/assets/en-US.pak
+- 3)       -130 (116.1%) p@0x0        -9 (9->0)  num_aliases=1
+               source_path= 	object_path=
+               flags={}  name=IDS_WEB_FONT_SIZE
+                    full_name=../../ui/strings/app_locale_settings.grd: IDS_WEB_FONT_SIZE
 ~ 4)       -120 (107.1%) r@0x284e398  +10 (22->32)  num_aliases=1
                source_path=third_party/container/container.c 	object_path=third_party/sub/ContiguousContainer.o
                flags={}  name=chrome::mojom::FilePatcher::Name_