Supersize: Track compressed sizes for pak symbols

Switch to tracking estimated compressed sizes for pak symbols as this
has a bigger impact to our apk size. Eventually we want to be able to
attribute every byte in our apk.

BUG=793015

Change-Id: I7a91c71a5d21f821c70aefcce6bffb2978ff157e
Reviewed-on: https://chromium-review.googlesource.com/822890
Commit-Queue: Peter Wen <wnwen@chromium.org>
Reviewed-by: agrieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#523480}

tools/binary_size/libsupersize/archive.py

@@ -692,8 +692,8 @@ def _ParseElfInfo(map_path, elf_path, tool_prefix, output_directory,
 
 
 def _ComputePakFileSymbols(
-    file_name, file_size, contents, res_info, symbols_by_id):
-  total = 12 + 6  # Header size plus extra offset
+    file_name, contents, res_info, symbols_by_id, expected_size,
+    compression_ratio=1):
   id_map = {id(v): k
             for k, v in sorted(contents.resources.items(), reverse=True)}
   alias_map = {k: id_map[id(v)] for k, v in contents.resources.iteritems()
@@ -703,6 +703,10 @@ def _ComputePakFileSymbols(
     section_name = models.SECTION_PAK_TRANSLATIONS
   else:
     section_name = models.SECTION_PAK_NONTRANSLATED
+  overhead = (12 + 6) * compression_ratio  # Header size plus extra offset
+  symbols_by_id[file_name] = models.Symbol(
+      section_name, overhead, full_name='{}: overhead'.format(file_name))
+  total = overhead
   for resource_id in sorted(contents.resources):
     if resource_id in alias_map:
       # 4 extra bytes of metadata (2 16-bit ints)
@@ -716,10 +720,12 @@ def _ComputePakFileSymbols(
         full_name = '{}: {}'.format(source_path, name)
         symbols_by_id[resource_id] = models.Symbol(
             section_name, 0, address=resource_id, full_name=full_name)
+    size *= compression_ratio
     symbols_by_id[resource_id].size += size
     total += size
-  assert file_size == total, (
-      '{} bytes in pak file not accounted for'.format(file_size - total))
+  total = int(round(total))
+  assert expected_size == total, (
+      '{} bytes in pak file not accounted for'.format(expected_size - total))
 
 
 def _ParsePakInfoFile(pak_info_path):
@@ -731,7 +737,8 @@ def _ParsePakInfoFile(pak_info_path):
   return res_info
 
 
-def _ParsePakSymbols(object_paths, output_directory, symbols_by_id):
+def _ParsePakSymbols(
+    section_sizes, object_paths, output_directory, symbols_by_id):
   for path in object_paths:
     whitelist_path = os.path.join(output_directory, path + '.whitelist')
     if (not os.path.exists(whitelist_path)
@@ -750,11 +757,23 @@ def _ParsePakSymbols(object_paths, output_directory, symbols_by_id):
 
   raw_symbols = sorted(symbols_by_id.values(),
                        key=lambda s: (s.section_name, s.address))
-  section_sizes = {}
+  raw_total = 0.0
+  int_total = 0
+  for symbol in raw_symbols:
+    raw_total += symbol.size
+    # We truncate rather than round to ensure that we do not over attribute. It
+    # is easier to add another symbol to make up the difference.
+    symbol.size = int(symbol.size)
+    int_total += symbol.size
+  # Attribute excess to translations since only those are compressed.
+  raw_symbols.append(models.Symbol(
+      models.SECTION_PAK_TRANSLATIONS, int(round(raw_total - int_total)),
+      full_name='Pak compression leftover artifacts'))
+
   for symbol in raw_symbols:
     prev = section_sizes.setdefault(symbol.section_name, 0)
     section_sizes[symbol.section_name] = prev + symbol.size
-  return section_sizes, raw_symbols
+  return raw_symbols
 
 
 def _ParseApkElfSectionSize(section_sizes, metadata, apk_elf_result):
@@ -793,7 +812,8 @@ def _ParseApkOtherSymbols(section_sizes, apk_path):
       apk_symbols.append(models.Symbol(
             models.SECTION_OTHER, zip_info.compress_size,
             full_name=zip_info.filename))
-  section_sizes[models.SECTION_OTHER] = sum(s.size for s in apk_symbols)
+  prev = section_sizes.setdefault(models.SECTION_OTHER, 0)
+  section_sizes[models.SECTION_OTHER] = prev + sum(s.size for s in apk_symbols)
   return apk_symbols
 
 
@@ -804,15 +824,17 @@ def _FindPakSymbolsFromApk(apk_path, output_directory):
     pak_info_path = os.path.join(output_directory, 'size-info', apk_info_name)
     res_info = _ParsePakInfoFile(pak_info_path)
     symbols_by_id = {}
-    for pak_zip_info in pak_zip_infos:
-      contents = data_pack.ReadDataPackFromString(z.read(pak_zip_info))
+    for zip_info in pak_zip_infos:
+      contents = data_pack.ReadDataPackFromString(z.read(zip_info))
+      compression_ratio = float(zip_info.compress_size) / zip_info.file_size
       _ComputePakFileSymbols(
-          pak_zip_info.filename, pak_zip_info.file_size, contents, res_info,
-          symbols_by_id)
+          os.path.relpath(zip_info.filename, output_directory), contents,
+          res_info, symbols_by_id, expected_size=zip_info.compress_size,
+          compression_ratio=compression_ratio)
   return symbols_by_id
 
 
-def _FindPakSymbolsFromFiles(pak_files, pak_info_path):
+def _FindPakSymbolsFromFiles(pak_files, pak_info_path, output_directory):
   """Uses files from args to find and add pak symbols."""
   res_info = _ParsePakInfoFile(pak_info_path)
   symbols_by_id = {}
@@ -820,8 +842,8 @@ def _FindPakSymbolsFromFiles(pak_files, pak_info_path):
     with open(pak_file_path, 'r') as f:
       contents = data_pack.ReadDataPackFromString(f.read())
       _ComputePakFileSymbols(
-          pak_file_path, os.path.getsize(pak_file_path), contents, res_info,
-          symbols_by_id)
+          os.path.relpath(pak_file_path, output_directory), contents, res_info,
+          symbols_by_id, expected_size=os.path.getsize(pak_file_path))
   return symbols_by_id
 
 
@@ -861,17 +883,17 @@ def CreateSectionSizesAndSymbols(
   pak_symbols_by_id = None
   if apk_path:
     pak_symbols_by_id = _FindPakSymbolsFromApk(apk_path, output_directory)
-    section_sizes = _ParseApkElfSectionSize(section_sizes, metadata,
-                                            apk_elf_result)
+    section_sizes = _ParseApkElfSectionSize(
+        section_sizes, metadata, apk_elf_result)
     raw_symbols.extend(_ParseApkOtherSymbols(section_sizes, apk_path))
   elif pak_files and pak_info_file:
-    pak_symbols_by_id = _FindPakSymbolsFromFiles(pak_files, pak_info_file)
+    pak_symbols_by_id = _FindPakSymbolsFromFiles(
+        pak_files, pak_info_file, output_directory)
 
   if pak_symbols_by_id:
     object_paths = (p for p in source_mapper.IterAllPaths() if p.endswith('.o'))
-    pak_section_sizes, pak_raw_symbols = _ParsePakSymbols(
-        object_paths, output_directory, pak_symbols_by_id)
-    section_sizes.update(pak_section_sizes)
+    pak_raw_symbols = _ParsePakSymbols(
+        section_sizes, object_paths, output_directory, pak_symbols_by_id)
     raw_symbols.extend(pak_raw_symbols)
 
   _ExtractSourcePathsAndNormalizeObjectPaths(raw_symbols, source_mapper)

tools/binary_size/libsupersize/testdata/Archive_Pak.golden

@@ -31,7 +31,7 @@ Section .bss: has 40.3% of 524520 bytes accounted for from 6 symbols. 775936 byt
 * Padding accounts for 196 bytes (0.0%)
 * Contains 0 aliases
 * 0 symbols have shared ownership
-Section .pak.translations: has 100.0% of 6803 bytes accounted for from 205 symbols. 0 bytes are unaccounted for.
+Section .pak.translations: has 100.0% of 6821 bytes accounted for from 207 symbols. 0 bytes are unaccounted for.
 * Padding accounts for 0 bytes (0.0%)
 * Contains 0 aliases
 * 0 symbols have shared ownership
@@ -87,6 +87,8 @@ Section .pak.translations: has 100.0% of 6803 bytes accounted for from 205 symbo
 .bss@2dffda0(size_without_padding=28,padding=0,full_name=g_chrome_content_browser_client,object_path=third_party/icu/icuuc/ucnv_ext.o,source_path=third_party/icu/ucnv_ext.c,flags={gen},num_aliases=1)
 .bss@2dffe80(size_without_padding=4,padding=196,full_name=SaveHistogram(_JNIEnv*, base::android::JavaParamRef<_jobject*> const&, base::android::JavaParamRef<_jstring*> const&, base::android::JavaParamRef<_jlongArray*> const&, int)::atomic_histogram_pointer,object_path=third_party/icu/icuuc/ucnv_ext.o,source_path=third_party/icu/ucnv_ext.c,flags={gen},num_aliases=1)
 .bss@2dffe84(size_without_padding=4,padding=0,full_name=g_AnimationFrameTimeHistogram_clazz,object_path=third_party/icu/icuuc/ucnv_ext.o,source_path=third_party/icu/ucnv_ext.c,flags={anon,gen},num_aliases=1)
+.pak.translations@0(size_without_padding=18,padding=0,full_name=en-US.pak: overhead,object_path=,source_path=,flags={},num_aliases=1)
+.pak.translations@0(size_without_padding=0,padding=0,full_name=Pak compression leftover artifacts,object_path=,source_path=,flags={},num_aliases=1)
 .pak.translations@3ac6(size_without_padding=40,padding=0,full_name=../../components/app_modal_strings.grdp: IDS_BEFOREUNLOAD_MESSAGEBOX_MESSAGE,object_path=,source_path=,flags={},num_aliases=1)
 .pak.translations@3d7c(size_without_padding=24,padding=0,full_name=../../components/policy_strings.grdp: IDS_POLICY_TYPE_ERROR,object_path=,source_path=,flags={},num_aliases=1)
 .pak.translations@3d82(size_without_padding=25,padding=0,full_name=../../components/policy_strings.grdp: IDS_POLICY_LIST_ENTRY_ERROR,object_path=,source_path=,flags={},num_aliases=1)