Commit fc8e613a authored by Andrew Grieve's avatar Andrew Grieve Committed by Commit Bot

Revert "[SuperSize] Add multiple Container support using .ssargs files."

This reverts commit 19d7a780.

Reason for revert: Breaking CQ (see bug)

Original change's description:
> [SuperSize] Add multiple Container support using .ssargs files.
> 
> This CL adds SuperSize support for multiple containers. Details:
> * Introduce .ssargs file, which is a text file to specify multiple
>   containers for SuperSize-archive to process into a single .size file.
>   * Container file types are auto-detected.
>   * Container files are relative to .ssargs file location, unless
>     absolute paths are given.
> * Introduce Version 1.1 .size file format:
>   * Header fields contain Container specs, now absorbing metadata and
>     section_sizes.
>   * Per-section data become per-container-per-section.
> * For compatibility: Running SuperSize-archive on one container still
>   generates Version 1.0 .size file format.
> * Update SuperSize-console output (including diffs) to accommodate.
> 
> After this CL, basic Trichrome support is available. A recommended
> .ssargs file (3 lines) for the official build is as follows:
> 
> TrichromeLibraryGoogle.apk --name Library
> TrichromeChromeGoogle.minimal.apks --name Chrome --java-only
> TrichromeWebViewGoogle.apk --name WebView --java-only
> 
> Bug: 1040645
> Change-Id: I67df00731d07660163410b59c57854bf56ea651e
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2248860
> Commit-Queue: Samuel Huang <huangs@chromium.org>
> Reviewed-by: Andrew Grieve <agrieve@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#780594}

TBR=huangs@chromium.org,agrieve@chromium.org

Change-Id: Ic3175ea3e8ede41e3183c12b5a00a4a09be55a75
No-Try: True
Bug: 1040645, 1097572
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2256593Reviewed-by: default avatarAndrew Grieve <agrieve@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#780639}
parent 540e2267
This diff is collapsed.
This diff is collapsed.
......@@ -150,25 +150,19 @@ def _DiffObj(before_obj, after_obj):
def _DiffContainerLists(before_containers, after_containers):
"""Computes diff of Containers lists, matching names."""
# Find ordered unique names, preferring order of |container_after|.
pairs = collections.OrderedDict()
for c in after_containers:
pairs[c.name] = [models.Container.Empty(), c]
for c in before_containers:
if c.name in pairs:
pairs[c.name][0] = c
else:
pairs[c.name] = [c, models.Container.Empty()]
"""Computes diffs between two lists of Containers."""
# TODO(huangs): Add support for multiple containers (needs name matching).
assert len(before_containers) == 1
assert len(after_containers) == 1
ret = []
for name, [before_c, after_c] in pairs.items():
for (before_c, after_c) in zip(before_containers, after_containers):
name = after_c.name
assert before_c.name == name
ret.append(
models.Container(name=name,
metadata=_DiffObj(before_c.metadata, after_c.metadata),
section_sizes=_DiffObj(before_c.section_sizes,
after_c.section_sizes)))
# This update newly created diff Containers, not existing ones or EMPTY.
models.Container.AssignShortNames(ret)
return ret
......
......@@ -37,7 +37,6 @@ def _CreateSizeInfo(aliases=None):
containers = [
models.Container(name='', metadata=metadata, section_sizes=section_sizes)
]
models.Container.AssignShortNames(containers)
TEXT = models.SECTION_TEXT
symbols = [
_MakeSym(models.SECTION_DEX_METHOD, 10, 'a', 'com.Foo#bar()'),
......
......@@ -5,9 +5,9 @@
"""Deals with loading & saving .size and .sizediff files.
The .size file is written in the following format. There are no section
delimiters, instead the end of a section is usually determined by a row count on
the first line of a section, followed by that number of rows. In other cases,
the sections have a known size.
delimiters, instead the end of a section is usually determined by a row count
on the first line of a section, followed by that number of rows. In other
cases, the sections have a known size.
Header
------
......@@ -116,8 +116,7 @@ import parallel
# File format version for .size files.
_SERIALIZATION_VERSION_SINGLE_CONTAINER = 'Size File Format v1'
_SERIALIZATION_VERSION_MULTI_CONTAINER = 'Size File Format v1.1'
_SERIALIZATION_VERSION = 'Size File Format v1'
# Header for .sizediff files
_SIZEDIFF_HEADER = '# Created by //tools/binary_size\nDIFF\n'
......@@ -172,18 +171,18 @@ def CalculatePadding(raw_symbols):
# Padding not really required, but it is useful to check for large padding and
# log a warning.
seen_container_and_sections = set()
# TODO(huangs): Add support for multiple containers: Need to group by
# container names and sections.
seen_sections = set()
for i, symbol in enumerate(raw_symbols[1:]):
prev_symbol = raw_symbols[i]
if symbol.IsOverhead():
# Overhead symbols are not actionable so should be padding-only.
symbol.padding = symbol.size
if (prev_symbol.container.name != symbol.container.name
or prev_symbol.section_name != symbol.section_name):
container_and_section = (symbol.container.name, symbol.section_name)
assert container_and_section not in seen_container_and_sections, (
'Input symbols must be sorted by container, section, then address.')
seen_container_and_sections.add(container_and_section)
if prev_symbol.section_name != symbol.section_name:
assert symbol.section_name not in seen_sections, (
'Input symbols must be sorted by section, then address.')
seen_sections.add(symbol.section_name)
continue
if (symbol.address <= 0 or prev_symbol.address <= 0
or not symbol.IsNative() or not prev_symbol.IsNative()):
......@@ -249,34 +248,21 @@ def _SaveSizeInfoToFile(size_info,
else:
raw_symbols = size_info.raw_symbols
num_containers = len(size_info.containers)
has_multi_containers = (num_containers > 1)
w = _Writer(file_obj)
# "Created by SuperSize" header
# Created by supersize header
w.WriteLine('# Created by //tools/binary_size')
if has_multi_containers:
w.WriteLine(_SERIALIZATION_VERSION_MULTI_CONTAINER)
else:
w.WriteLine(_SERIALIZATION_VERSION_SINGLE_CONTAINER)
w.WriteLine(_SERIALIZATION_VERSION)
# JSON header fields
fields = {
'has_components': True,
'has_padding': include_padding,
}
num_containers = len(size_info.containers)
has_multi_containers = (num_containers > 1)
if has_multi_containers:
# Write using new format.
assert len(set(c.name for c in size_info.containers)) == num_containers, (
'Container names must be distinct.')
fields['build_config'] = size_info.build_config
fields['containers'] = [{
'name': c.name,
'metadata': c.metadata,
'section_sizes': c.section_sizes,
} for c in size_info.containers]
raise ValueError('Multiple container not yet supported.')
else:
# Write using old format.
fields['metadata'] = size_info.metadata_legacy
......@@ -304,18 +290,12 @@ def _SaveSizeInfoToFile(size_info,
w.WriteLine(comp)
w.LogSize('components')
# Symbol counts by container and section.
symbol_group_by_section = raw_symbols.GroupedByContainerAndSectionName()
# Symbol counts by section.
symbol_group_by_section = raw_symbols.GroupedBySectionName()
if has_multi_containers:
container_name_to_index = {
c.name: i
for i, c in enumerate(size_info.containers)
}
w.WriteLine('\t'.join('<%d>%s' %
(container_name_to_index[g.name[0]], g.name[1])
for g in symbol_group_by_section))
raise ValueError('Multiple container not yet supported.')
else:
w.WriteLine('\t'.join(g.name[1] for g in symbol_group_by_section))
w.WriteLine('\t'.join(g.name for g in symbol_group_by_section))
w.WriteLine('\t'.join(str(len(g)) for g in symbol_group_by_section))
def gen_delta(gen, prev_value=0):
......@@ -404,35 +384,23 @@ def _LoadSizeInfoFromFile(file_obj, size_path):
"""
# Split lines on '\n', since '\r' can appear in some lines!
lines = io.TextIOWrapper(file_obj, newline='\n')
_ReadLine(lines) # Line 0: "Created by SuperSize" header
_ReadLine(lines) # Line 0: Created by supersize header
actual_version = _ReadLine(lines)
if actual_version == _SERIALIZATION_VERSION_SINGLE_CONTAINER:
has_multi_containers = False
elif actual_version == _SERIALIZATION_VERSION_MULTI_CONTAINER:
has_multi_containers = True
else:
raise ValueError('Version mismatch. Need to write some upgrade code.')
assert actual_version == _SERIALIZATION_VERSION, (
'Version mismatch. Need to write some upgrade code.')
# JSON header fields
json_len = int(_ReadLine(lines))
json_str = lines.read(json_len)
fields = json.loads(json_str)
assert ('containers' in fields) == has_multi_containers
assert ('build_config' in fields) == has_multi_containers
assert ('containers' in fields) == has_multi_containers
assert ('metadata' not in fields) == has_multi_containers
assert ('section_sizes' not in fields) == has_multi_containers
has_multi_containers = False
containers = []
if has_multi_containers: # New format.
build_config = fields['build_config']
for cfield in fields['containers']:
c = models.Container(name=cfield['name'],
metadata=cfield['metadata'],
section_sizes=cfield['section_sizes'])
containers.append(c)
else: # Old format.
raise ValueError('Multiple container not yet supported.')
else:
# Parse old format, but separate data into build_config and metadata.
build_config = {}
metadata = fields.get('metadata')
if metadata:
......@@ -445,7 +413,6 @@ def _LoadSizeInfoFromFile(file_obj, size_path):
models.Container(name='',
metadata=metadata,
section_sizes=section_sizes))
models.Container.AssignShortNames(containers)
has_components = fields.get('has_components', False)
has_padding = fields.get('has_padding', False)
......@@ -466,7 +433,7 @@ def _LoadSizeInfoFromFile(file_obj, size_path):
components = [_ReadLine(lines) for _ in range(num_components)]
# Symbol counts by section.
container_and_section_names = _ReadValuesFromLine(lines, split='\t')
section_names = _ReadValuesFromLine(lines, split='\t')
symbol_counts = [int(c) for c in _ReadValuesFromLine(lines, split='\t')]
# Addresses, sizes, paddings, path indices, component indices
......@@ -493,28 +460,23 @@ def _LoadSizeInfoFromFile(file_obj, size_path):
if has_padding:
paddings = read_numeric(delta=False)
else:
paddings = [None] * len(container_and_section_names)
paddings = [None] * len(section_names)
path_indices = read_numeric(delta=True)
if has_components:
component_indices = read_numeric(delta=True)
else:
component_indices = [None] * len(container_and_section_names)
component_indices = [None] * len(section_names)
raw_symbols = [None] * sum(symbol_counts)
symbol_idx = 0
for (cur_container_and_section_name, cur_symbol_count, cur_addresses,
cur_sizes, cur_paddings, cur_path_indices,
cur_component_indices) in zip(container_and_section_names, symbol_counts,
addresses, sizes, paddings, path_indices,
for (cur_section_name, cur_symbol_count, cur_addresses, cur_sizes,
cur_paddings, cur_path_indices,
cur_component_indices) in zip(section_names, symbol_counts, addresses,
sizes, paddings, path_indices,
component_indices):
if has_multi_containers:
# Extract '<cur_container_idx_str>cur_section_name'.
assert cur_container_and_section_name.startswith('<')
cur_container_idx_str, cur_section_name = (
cur_container_and_section_name[1:].split('>', 1))
cur_container = containers[int(cur_container_idx_str)]
raise ValueError('Multiple container not yet supported.')
else:
cur_section_name = cur_container_and_section_name
cur_container = containers[0]
alias_counter = 0
for i in range(cur_symbol_count):
......
......@@ -309,10 +309,7 @@ class IntegrationTest(unittest.TestCase):
merged_data_desc = describe.DescribeDict(size_info.metadata_legacy)
return itertools.chain(merged_data_desc, stats, sym_strs)
else:
build_config = describe.DescribeDict(size_info.build_config)
metadata = itertools.chain.from_iterable(
describe.DescribeDict(c.metadata) for c in size_info.containers)
return itertools.chain(build_config, metadata, stats, sym_strs)
raise ValueError('Multiple container not yet supported.')
@_CompareWithGolden()
def test_Archive(self):
......
......@@ -102,8 +102,6 @@ PAK_SECTIONS = (
SECTION_PAK_TRANSLATIONS,
)
CONTAINER_MULTIPLE = '*'
SECTION_NAME_TO_SECTION = {
SECTION_BSS: 'b',
SECTION_BSS_REL_RO: 'b',
......@@ -203,19 +201,16 @@ def ClassifySections(section_names):
class Container(object):
"""Info for a single SuperSize input file (e.g., APK file).
"""Info for a a single input file for SuperSize, e.g., an APK file.
Fields:
name: Container name. Must be unique among containers, and can be ''.
short_name: Short container name for compact display. This, also needs to be
unique among containers in the same SizeInfo, and can be ''.
metadata: A dict.
section_sizes: A dict of section_name -> size.
classified_sections: Cache for ClassifySections().
"""
__slots__ = (
'name',
'short_name',
'metadata',
'section_sizes',
'_classified_sections',
......@@ -225,31 +220,15 @@ class Container(object):
# name == '' hints that only one container exists, and there's no need to
# distinguish them. This can affect console output.
self.name = name
self.short_name = None # Assigned by AssignShortNames().
self.metadata = metadata or {}
self.section_sizes = section_sizes # E.g. {SECTION_TEXT: 0}
self._classified_sections = None
@staticmethod
def AssignShortNames(containers):
for i, c in enumerate(containers):
c.short_name = str(i) if c.name else ''
def ClassifySections(self):
if not self._classified_sections:
self._classified_sections = ClassifySections(self.section_sizes.keys())
return self._classified_sections
@staticmethod
def Empty():
"""Returns a placeholder Container that should be read-only.
For simplicity, we're not enforcing read-only checks (frozenmap does not
exist, unfortunately). Creating a new instance instead of using a global
singleton for robustness.
"""
return Container(name='(empty)', metadata={}, section_sizes={})
class BaseSizeInfo(object):
"""Base class for SizeInfo and DeltaSizeInfo.
......@@ -282,7 +261,6 @@ class BaseSizeInfo(object):
self._symbols = symbols
self._native_symbols = None
self._pak_symbols = None
Container.AssignShortNames(self.containers)
@property
def symbols(self):
......@@ -318,9 +296,6 @@ class BaseSizeInfo(object):
def metadata(self):
return [c.metadata for c in self.containers]
def ContainerForName(self, name, default=None):
return next((c for c in containers if c.name == name), default)
class SizeInfo(BaseSizeInfo):
"""Represents all size information for a single binary.
......@@ -387,10 +362,6 @@ class BaseSymbol(object):
def container_name(self):
return self.container.name if self.container else ''
@property
def container_short_name(self):
return self.container.short_name if self.container else ''
@property
def section(self):
"""Returns the one-letter section."""
......@@ -545,17 +516,14 @@ class Symbol(BaseSymbol):
self.component = ''
def __repr__(self):
if self.container and self.container.name:
container_str = '<{}>'.format(self.container.name)
else:
container_str = ''
template = ('{}{}@{:x}(size_without_padding={},padding={},full_name={},'
# TODO(huangs): If container name is nonempty then display it.
template = ('{}@{:x}(size_without_padding={},padding={},full_name={},'
'object_path={},source_path={},flags={},num_aliases={},'
'component={})')
return template.format(container_str, self.section_name, self.address,
self.size_without_padding, self.padding,
self.full_name, self.object_path, self.source_path,
self.FlagsString(), self.num_aliases, self.component)
return template.format(
self.section_name, self.address, self.size_without_padding,
self.padding, self.full_name, self.object_path, self.source_path,
self.FlagsString(), self.num_aliases, self.component)
def SetName(self, full_name, template_name=None, name=None):
# Note that _NormalizeNames() will clobber these values.
......@@ -594,21 +562,14 @@ class DeltaSymbol(BaseSymbol):
self.after_symbol = after_symbol
def __repr__(self):
before_container_name = self.before_symbol.container_name
after_container_name = self.after_symbol.container_name
if before_container_name != after_container_name:
container_str = '<~{}>'.format(after_container_name)
elif after_container_name != '':
container_str = '<{}>'.format(after_container_name)
else:
container_str = ''
template = ('{}{}{}@{:x}(size_without_padding={},padding={},full_name={},'
# TODO(huangs): If container name is nonempty then display it.
template = ('{}{}@{:x}(size_without_padding={},padding={},full_name={},'
'object_path={},source_path={},flags={})')
return template.format(DIFF_PREFIX_BY_STATUS[self.diff_status],
container_str, self.section_name, self.address,
self.size_without_padding, self.padding,
self.full_name, self.object_path, self.source_path,
self.FlagsString())
return template.format(
DIFF_PREFIX_BY_STATUS[self.diff_status], self.section_name,
self.address, self.size_without_padding, self.padding,
self.full_name, self.object_path, self.source_path,
self.FlagsString())
def IsDelta(self):
return True
......@@ -751,11 +712,11 @@ class SymbolGroup(BaseSymbol):
'full_name',
'template_name',
'name',
'container_name',
'section_name',
'is_default_sorted', # True for groups created by Sorted()
)
# template_name and full_name are useful when clustering symbol clones.
def __init__(self,
symbols,
......@@ -774,6 +735,8 @@ class SymbolGroup(BaseSymbol):
self.full_name = full_name if full_name is not None else name
self.template_name = template_name if template_name is not None else name
self.name = name or ''
# TODO(huangs): Add support for multiple containers.
self.container_name = ''
self.section_name = section_name or SECTION_MULTIPLE
self.is_default_sorted = is_default_sorted
......@@ -820,20 +783,6 @@ class SymbolGroup(BaseSymbol):
def index(self, item):
return self._symbols.index(item)
@property
def container_name(self):
ret = set(s.container_name for s in self._symbols)
if ret:
return CONTAINER_MULTIPLE if len(ret) > 1 else (ret.pop() or '')
return ''
@property
def container_short_name(self):
ret = set(s.container_short_name for s in self._symbols)
if ret:
return CONTAINER_MULTIPLE if len(ret) > 1 else (ret.pop() or '')
return ''
@property
def address(self):
first = self._symbols[0].address if self else 0
......@@ -989,23 +938,13 @@ class SymbolGroup(BaseSymbol):
def WherePssBiggerThan(self, min_pss):
return self.Filter(lambda s: s.pss >= min_pss)
def WhereInSection(self, section, container=None):
def WhereInSection(self, section):
"""|section| can be section_name ('.bss'), or section chars ('bdr')."""
if section.startswith('.'):
if container:
short_name = container.short_name
ret = self.Filter(lambda s: (s.container.short_name == short_name and s.
section_name == section))
else:
ret = self.Filter(lambda s: s.section_name == section)
ret = self.Filter(lambda s: s.section_name == section)
ret.section_name = section
else:
if container:
short_name = container.short_name
ret = self.Filter(lambda s: (s.container.short_name == short_name and s.
section in section))
else:
ret = self.Filter(lambda s: s.section in section)
ret = self.Filter(lambda s: s.section in section)
if section in SECTION_TO_SECTION_NAME:
ret.section_name = SECTION_TO_SECTION_NAME[section]
return ret
......@@ -1254,9 +1193,6 @@ class SymbolGroup(BaseSymbol):
lambda s: (same_name_only and s.full_name, id(s.aliases or s)),
min_count=min_count, group_factory=group_factory)
def GroupedByContainerAndSectionName(self):
return self.GroupedBy(lambda s: (s.container.name, s.section_name))
def GroupedBySectionName(self):
return self.GroupedBy(lambda s: s.section_name)
......
********************************************************************************
Entering interactive Python shell. Quick reference:
SizeInfo: ContainerForName, all_section_sizes, build_config, containers, metadata, metadata_legacy, native_symbols, pak_symbols, raw_symbols, size_path, symbols
Symbol: FlagsString, IsBss, IsDelta, IsDex, IsGeneratedByToolchain, IsGroup, IsNameUnique, IsNative, IsOther, IsOverhead, IsPak, IsStringLiteral, IterLeafSymbols, SetName, address, aliases, component, container, container_name, container_short_name, end_address, flags, full_name, generated_source, is_anonymous, name, num_aliases, object_path, padding, padding_pss, pss, pss_without_padding, section, section_name, size, size_without_padding, source_path, template_name
SizeInfo: all_section_sizes, build_config, containers, metadata, metadata_legacy, native_symbols, pak_symbols, raw_symbols, size_path, symbols
Symbol: FlagsString, IsBss, IsDelta, IsDex, IsGeneratedByToolchain, IsGroup, IsNameUnique, IsNative, IsOther, IsOverhead, IsPak, IsStringLiteral, IterLeafSymbols, SetName, address, aliases, component, container, container_name, end_address, flags, full_name, generated_source, is_anonymous, name, num_aliases, object_path, padding, padding_pss, pss, pss_without_padding, section, section_name, size, size_without_padding, source_path, template_name
SymbolGroup (extends Symbol): CountUniqueSymbols, Filter, GroupedBy, GroupedByAliases, GroupedByComponent, GroupedByContainerAndSectionName, GroupedByFullName, GroupedByName, GroupedByPath, GroupedBySectionName, Inverted, IterUniqueSymbols, Sorted, SortedByAddress, SortedByCount, SortedByName, WhereAddressInRange, WhereComponentMatches, WhereFullNameMatches, WhereGeneratedByToolchain, WhereHasAnyAttribution, WhereHasComponent, WhereHasFlag, WhereHasPath, WhereInSection, WhereIsDex, WhereIsGroup, WhereIsNative, WhereIsPak, WhereIsTemplate, WhereMatches, WhereNameMatches, WhereObjectPathMatches, WherePathMatches, WherePssBiggerThan, WhereSizeBiggerThan, WhereSourceIsGenerated, WhereSourcePathMatches, WhereTemplateNameMatches, index, is_default_sorted
SymbolGroup (extends Symbol): CountUniqueSymbols, Filter, GroupedBy, GroupedByAliases, GroupedByComponent, GroupedByFullName, GroupedByName, GroupedByPath, GroupedBySectionName, Inverted, IterUniqueSymbols, Sorted, SortedByAddress, SortedByCount, SortedByName, WhereAddressInRange, WhereComponentMatches, WhereFullNameMatches, WhereGeneratedByToolchain, WhereHasAnyAttribution, WhereHasComponent, WhereHasFlag, WhereHasPath, WhereInSection, WhereIsDex, WhereIsGroup, WhereIsNative, WhereIsPak, WhereIsTemplate, WhereMatches, WhereNameMatches, WhereObjectPathMatches, WherePathMatches, WherePssBiggerThan, WhereSizeBiggerThan, WhereSourceIsGenerated, WhereSourcePathMatches, WhereTemplateNameMatches, index, is_default_sorted
DeltaSizeInfo: ContainerForName, after, all_section_sizes, before, build_config, containers, metadata, native_symbols, pak_symbols, raw_symbols, symbols
DeltaSizeInfo: after, all_section_sizes, before, build_config, containers, metadata, native_symbols, pak_symbols, raw_symbols, symbols
DeltaSymbol (extends Symbol): after_symbol, before_symbol, diff_status
DeltaSymbolGroup (extends SymbolGroup): CountsByDiffStatus, WhereDiffStatusIs, diff_status
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment