Skip to content

GitLab

T
tangled
Commit 8b141623 authored by Matthew Cary's avatar Matthew Cary Committed by Commit Bot
Browse files
Options

Orderfile: Only use symbols in orderfile generation 

Removes gold-style section orderfile, as that is now not possible to
generate with ThinLTO in lld.

Bug: 869997
Change-Id: I9d6970cf9c251cefdc69fbef6c87118fde844374
Reviewed-on: https://chromium-review.googlesource.com/1162221
Commit-Queue: Matthew Cary <mattcary@chromium.org>
Reviewed-by: default avatarEgor Pasko <pasko@chromium.org>
Cr-Commit-Position: refs/heads/master@{#582282}
parent 2e49a8d6
Hide whitespace changes
Inline Side-by-side
Showing with 105 additions and 530 deletions
tools/cygprofile/check_orderfile.py
View file @ 8b141623
......@@ -16,58 +16,6 @@ import patch_orderfile
import symbol_extractor
_MAX_WARNINGS_TO_PRINT = 200
def _IsSameMethod(name1, name2):
"""Returns true if name1 or name2 are split method forms of the other."""
return patch_orderfile.RemoveSuffixes(name1) == \
patch_orderfile.RemoveSuffixes(name2)
def _CountMisorderedSymbols(symbols, symbol_infos):
"""Count the number of misordered symbols, and log them.
Args:
symbols: ordered sequence of symbols from the orderfile
symbol_infos: ordered list of SymbolInfo from the binary
Returns:
(misordered_pairs_count, matched_symbols_count, unmatched_symbols_count)
"""
name_to_symbol_info = symbol_extractor.CreateNameToSymbolInfo(symbol_infos)
matched_symbol_infos = []
missing_count = 0
misordered_count = 0
# Find the SymbolInfo matching the orderfile symbols in the binary.
for symbol in symbols:
if symbol in name_to_symbol_info:
matched_symbol_infos.append(name_to_symbol_info[symbol])
else:
missing_count += 1
if missing_count < _MAX_WARNINGS_TO_PRINT:
logging.warning('Symbol "%s" is in the orderfile, not in the binary' %
symbol)
logging.info('%d matched symbols, %d un-matched (Only the first %d unmatched'
' symbols are shown)' % (
len(matched_symbol_infos), missing_count,
_MAX_WARNINGS_TO_PRINT))
# In the order of the orderfile, find all the symbols that are at an offset
# smaller than their immediate predecessor, and record the pair.
previous_symbol_info = symbol_extractor.SymbolInfo(
name='', offset=-1, size=0, section='')
for symbol_info in matched_symbol_infos:
if symbol_info.offset < previous_symbol_info.offset and not (
_IsSameMethod(symbol_info.name, previous_symbol_info.name)):
logging.warning('Misordered pair: %s - %s' % (
str(previous_symbol_info), str(symbol_info)))
misordered_count += 1
previous_symbol_info = symbol_info
return (misordered_count, len(matched_symbol_infos), missing_count)
def _VerifySymbolOrder(orderfile_symbols, symbol_infos):
"""Verify symbol ordering.
......@@ -98,6 +46,7 @@ def _VerifySymbolOrder(orderfile_symbols, symbol_infos):
logging.warning('Missing symbols in verification: %d', missing_count)
return True
def main():
parser = optparse.OptionParser(usage=
'usage: %prog [options] <binary> <orderfile>')
......@@ -115,25 +64,12 @@ def main():
(binary_filename, orderfile_filename) = argv[1:]
symbol_extractor.SetArchitecture(options.arch)
obj_dir = cygprofile_utils.GetObjDir(binary_filename)
symbol_to_sections_map = cyglog_to_orderfile.ObjectFileProcessor(
obj_dir).GetSymbolToSectionsMap()
section_to_symbols_map = cygprofile_utils.InvertMapping(
symbol_to_sections_map)
symbols = patch_orderfile.GetSymbolsFromOrderfile(orderfile_filename,
section_to_symbols_map)
symbol_infos = symbol_extractor.SymbolInfosFromBinary(binary_filename)
if not _VerifySymbolOrder([sym.strip() for sym in file(orderfile_filename)],
symbol_infos):
return 1
# Missing symbols is not an error since some of them can be eliminated through
# inlining.
(misordered_pairs_count, matched_symbols, _) = _CountMisorderedSymbols(
symbols, symbol_infos)
return (misordered_pairs_count > options.threshold) or (matched_symbols == 0)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
......
tools/cygprofile/check_orderfile_unittest.py
View file @ 8b141623
......@@ -15,32 +15,6 @@ class TestCheckOrderFile(unittest.TestCase):
symbol_extractor.SymbolInfo('notProfiled', 0x4, 0, ''),
symbol_extractor.SymbolInfo('third', 0x3, 0, ''),]
def testMatchesSymbols(self):
symbols = ['first', 'second', 'third']
(misordered_pairs_count, matched_count, missing_count) = (
check_orderfile._CountMisorderedSymbols(symbols, self._SYMBOL_INFOS))
self.assertEquals(
(misordered_pairs_count, matched_count, missing_count), (0, 3, 0))
def testMissingMatches(self):
symbols = ['second', 'third', 'other', 'first']
(_, matched_count, unmatched_count) = (
check_orderfile._CountMisorderedSymbols(symbols, self._SYMBOL_INFOS))
self.assertEquals(matched_count, 3)
self.assertEquals(unmatched_count, 1)
def testNoUnorderedSymbols(self):
symbols = ['first', 'other', 'second', 'third', 'noMatchEither']
(misordered_pairs_count, _, _) = (
check_orderfile._CountMisorderedSymbols(symbols, self._SYMBOL_INFOS))
self.assertEquals(misordered_pairs_count, 0)
def testUnorderedSymbols(self):
symbols = ['first', 'other', 'third', 'second', 'noMatchEither']
(misordered_pairs_count, _, _) = (
check_orderfile._CountMisorderedSymbols(symbols, self._SYMBOL_INFOS))
self.assertEquals(misordered_pairs_count, 1)
def testVerifySymbolOrder(self):
self.assertTrue(check_orderfile._VerifySymbolOrder(
['.second', 'first', 'eighth', 'third'],
......
tools/cygprofile/cyglog_to_orderfile.py
View file @ 8b141623
......@@ -256,18 +256,13 @@ def main():
args.arch = cygprofile_utils.DetectArchitecture()
symbol_extractor.SetArchitecture(args.target_arch)
obj_dir = cygprofile_utils.GetObjDir(args.native_library)
offsets = _ReadReachedOffsets(args.reached_offsets)
assert offsets
_WarnAboutDuplicates(offsets)
generator = OffsetOrderfileGenerator(
process_profiles.SymbolOffsetProcessor(args.native_library),
ObjectFileProcessor(obj_dir))
ordered_sections = generator.GetOrderedSections(offsets)
if ordered_sections is None:
processor = process_profiles.SymbolOffsetProcessor(args.native_library)
ordered_symbols = processor.GetOrderedSymbols(offsets)
if ordered_symbols is None:
return 1
success = False
......@@ -276,7 +271,7 @@ def main():
try:
(fd, temp_filename) = tempfile.mkstemp(dir=os.path.dirname(args.output))
output_file = os.fdopen(fd, 'w')
output_file.write('\n'.join(ordered_sections))
output_file.write('\n'.join(ordered_symbols))
output_file.close()
os.rename(temp_filename, args.output)
temp_filename = None
......
tools/cygprofile/orderfile_generator_backend.py
View file @ 8b141623
......@@ -236,12 +236,11 @@ class ClankCompiler(object):
self._max_load = max_load
self._use_goma = use_goma
self._goma_dir = goma_dir
lib_chrome_so_dir = 'lib.unstripped'
self._system_health_profiling = system_health_profiling
self.obj_dir = os.path.join(self._out_dir, 'Release', 'obj')
self.lib_chrome_so = os.path.join(
self._out_dir, 'Release', lib_chrome_so_dir, 'libchrome.so')
self._out_dir, 'Release', 'lib.unstripped', 'libchrome.so')
self.chrome_apk = os.path.join(
self._out_dir, 'Release', 'apks', 'Chrome.apk')
......@@ -401,9 +400,6 @@ class OrderfileGenerator(object):
generates an updated orderfile.
"""
_CLANK_REPO = os.path.join(constants.DIR_SOURCE_ROOT, 'clank')
_CYGLOG_TO_ORDERFILE_SCRIPT = os.path.join(
constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile',
'cyglog_to_orderfile.py')
_CHECK_ORDERFILE_SCRIPT = os.path.join(
constants.DIR_SOURCE_ROOT, 'tools', 'cygprofile', 'check_orderfile.py')
_BUILD_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(
......@@ -411,8 +407,6 @@ class OrderfileGenerator(object):
_UNPATCHED_ORDERFILE_FILENAME = os.path.join(
_CLANK_REPO, 'orderfiles', 'unpatched_orderfile.%s')
_MERGED_CYGLOG_FILENAME = os.path.join(
constants.GetOutDirectory(), 'merged_cyglog')
_PATH_TO_ORDERFILE = os.path.join(_CLANK_REPO, 'orderfiles',
'orderfile.%s.out')
......@@ -506,6 +500,7 @@ class OrderfileGenerator(object):
except Exception:
for f in files:
self._SaveForDebugging(f)
self._SaveForDebugging(self._compiler.lib_chrome_so)
raise
finally:
self._profiler.Cleanup()
......@@ -541,14 +536,11 @@ class OrderfileGenerator(object):
offsets_list = (profile_offsets.startup +
profile_offsets.common +
profile_offsets.interaction)
generator = cyglog_to_orderfile.OffsetOrderfileGenerator(
processor, cyglog_to_orderfile.ObjectFileProcessor(
self._compiler.obj_dir))
ordered_sections = generator.GetOrderedSections(offsets_list)
if not ordered_sections:
raise Exception('Failed to get ordered sections')
ordered_symbols = processor.GetOrderedSymbols(offsets_list)
if not ordered_symbols:
raise Exception('Failed to get ordered symbols')
with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
orderfile.write('\n'.join(ordered_sections))
orderfile.write('\n'.join(ordered_symbols))
def _CollectLegacyProfile(self):
try:
......@@ -563,8 +555,14 @@ class OrderfileGenerator(object):
if not offsets:
raise Exception('No profiler offsets found in {}'.format(
'\n'.join(files)))
with open(self._MERGED_CYGLOG_FILENAME, 'w') as f:
f.write('\n'.join(map(str, offsets)))
processor = process_profiles.SymbolOffsetProcessor(
self._compiler.lib_chrome_so)
ordered_symbols = processor.GetOrderedSymbols(offsets)
if not ordered_symbols:
raise Exception('No symbol names from offsets found in {}'.format(
'\n'.join(files)))
with open(self._GetUnpatchedOrderfileFilename(), 'w') as orderfile:
orderfile.write('\n'.join(ordered_symbols))
except Exception:
for f in files:
self._SaveForDebugging(f)
......@@ -572,19 +570,6 @@ class OrderfileGenerator(object):
finally:
self._profiler.Cleanup()
try:
command_args = [
'--target-arch=' + self._options.arch,
'--native-library=' + self._compiler.lib_chrome_so,
'--output=' + self._GetUnpatchedOrderfileFilename()]
command_args.append('--reached-offsets=' + self._MERGED_CYGLOG_FILENAME)
self._step_recorder.RunCommand(
[self._CYGLOG_TO_ORDERFILE_SCRIPT] + command_args)
except CommandError:
self._SaveForDebugging(self._MERGED_CYGLOG_FILENAME)
self._SaveForDebuggingWithOverwrite(self._compiler.lib_chrome_so)
raise
def _MaybeSaveProfile(self, files):
if self._options.profile_save_dir:
logging.info('Saving profiles to %s', self._options.profile_save_dir)
......@@ -592,13 +577,6 @@ class OrderfileGenerator(object):
shutil.copy(f, self._options.profile_save_dir)
logging.info('Saved profile %s', f)
def _DeleteTempFiles(self):
"""Deletes intermediate step output files."""
print 'Delete %s' % (
self._MERGED_CYGLOG_FILENAME)
if os.path.isfile(self._MERGED_CYGLOG_FILENAME):
os.unlink(self._MERGED_CYGLOG_FILENAME)
def _PatchOrderfile(self):
"""Patches the orderfile using clean version of libchrome.so."""
self._step_recorder.BeginStep('Patch Orderfile')
......@@ -721,7 +699,6 @@ class OrderfileGenerator(object):
self._MaybeArchiveOrderfile(self._GetUnpatchedOrderfileFilename())
profile_uploaded = True
finally:
self._DeleteTempFiles()
_StashOutputDirectory(self._instrumented_out_dir)
elif self._options.manual_symbol_offsets:
assert self._options.manual_libname
......
tools/cygprofile/patch_orderfile.py
View file @ 8b141623
......@@ -6,14 +6,11 @@
"""Patch an orderfile.
Starting with a list of symbols in a binary and an orderfile (ordered list of
sections), matches the symbols in the orderfile and augments each symbol with
symbols), matches the symbols in the orderfile and augments each symbol with
the symbols residing at the same address (due to having identical code). The
output is a list of section or symbols matching rules appropriate for the linker
option -section-ordering-file for gold and --symbol-ordering-file for lld. Both
linkers are fine with extra directives that aren't matched in the binary, so we
construct a file suitable for both, concatenating sections and symbols. We
assume that the unpatched orderfile is built for gold, that is, it only contains
sections.
output is a list of symbols appropriate for the linker
option --symbol-ordering-file for lld. Note this is not usable with gold (which
uses section names to order the binary).
Note: It is possible to have.
- Several symbols mapping to the same offset in the binary.
......@@ -25,10 +22,7 @@ The general pipeline is:
2. Get the symbol names from the orderfile
3. Find the orderfile symbol names in the symbols coming from the binary
4. For each symbol found, get all the symbols at the same address
5. Output them to an updated orderfile suitable for gold and lld
6. Output catch-all section matching rules for unprofiled methods. This is
ineffective for lld, as it doesn't handle wildcards, but puts unordered
symbols after the ordered ones.
5. Output them to an updated orderfile suitable lld
"""
import argparse
......@@ -40,13 +34,7 @@ import cyglog_to_orderfile
import cygprofile_utils
import symbol_extractor
# Prefixes for the symbols. We strip them from the incoming symbols, and add
# them back in the output file.
# Output sections are constructed as prefix + symbol_name, hence the empty
# prefix is used to generate the symbol entry for lld.
_PREFIXES = ('.text.hot.', '.text.unlikely.', '.text.', '')
# Suffixes for the symbols. These are due to method splitting for inlining and
# Suffixes for symbols. These are due to method splitting for inlining and
# method cloning for various reasons including constant propagation and
# inter-procedural optimization.
_SUFFIXES = ('.clone.', '.part.', '.isra.', '.constprop.')
......@@ -89,272 +77,49 @@ def _UniqueGenerator(generator):
return _FilteringFunction
def _GroupSymbolInfosFromBinary(binary_filename):
"""Group all the symbols from a binary by name and offset.
Args:
binary_filename: path to the binary.
def _GroupSymbolsByOffset(binary_filename):
"""Produce a map symbol name -> all symbol names at same offset.
Returns:
A tuple of dict:
(offset_to_symbol_infos, name_to_symbol_infos):
- offset_to_symbol_infos: {offset: [symbol_info1, ...]}
- name_to_symbol_infos: {name: [symbol_info1, ...]}
Suffixes are stripped.
"""
symbol_infos = symbol_extractor.SymbolInfosFromBinary(binary_filename)
symbol_infos_no_suffixes = [
s._replace(name=RemoveSuffixes(s.name)) for s in symbol_infos]
return (symbol_extractor.GroupSymbolInfosByOffset(symbol_infos_no_suffixes),
symbol_extractor.GroupSymbolInfosByName(symbol_infos_no_suffixes))
symbol_infos = [
s._replace(name=RemoveSuffixes(s.name))
for s in symbol_extractor.SymbolInfosFromBinary(binary_filename)]
offset_map = symbol_extractor.GroupSymbolInfosByOffset(symbol_infos)
missing_offsets = 0
sym_to_matching = {}
for sym in symbol_infos:
if sym.offset not in offset_map:
missing_offsets += 1
continue
matching = [s.name for s in offset_map[sym.offset]]
assert sym.name in matching
sym_to_matching[sym.name] = matching
return sym_to_matching
def _StripPrefix(line):
"""Strips the linker section name prefix from a symbol line.
Args:
line: a line from an orderfile, usually in the form:
.text.SymbolName
Returns:
The symbol, SymbolName in the example above.
"""
# Went away with GCC, make sure it doesn't come back, as the orderfile
# no longer contains it.
assert not line.startswith('.text.startup.')
for prefix in _PREFIXES:
if prefix and line.startswith(prefix):
return line[len(prefix):]
return line # Unprefixed case
def _StripSuffixes(section_list):
"""Remove all suffixes on items in a list of symbols."""
return [RemoveSuffixes(section) for section in section_list]
def _SectionNameToSymbols(section_name, section_to_symbols_map):
"""Yields all symbols which could be referred to by section_name.
If the section name is present in the map, the names in the map are returned.
Otherwise, any clone annotations and prefixes are stripped from the section
name and the remainder is returned.
"""
if (not section_name or
section_name == '.text' or
section_name.endswith('*')):
return # Don't return anything for catch-all sections
if section_name in section_to_symbols_map:
for symbol in section_to_symbols_map[section_name]:
yield symbol
else:
name = _StripPrefix(section_name)
if name:
yield name
def GetSectionsFromOrderfile(filename):
"""Yields the sections from an orderfile.
@_UniqueGenerator
def ReadOrderfile(orderfile):
"""Reads an orderfile and cleans up symbols.
Args:
filename: The name of the orderfile.
orderfile: The name of the orderfile.
Yields:
A list of symbol names.
Symbol names, cleaned and unique.
"""
with open(filename, 'r') as f:
with open(orderfile) as f:
for line in f.xreadlines():
line = line.rstrip('\n')
line = line.strip()
if line:
yield line
@_UniqueGenerator
def GetSymbolsFromOrderfile(filename, section_to_symbols_map):
"""Yields the symbols from an orderfile. Output elements do not repeat.
Args:
filename: The name of the orderfile.
section_to_symbols_map: The mapping from section to symbol names. If a
section name is missing from the mapping, the
symbol name is assumed to be the section name with
prefixes and suffixes stripped.
Yields:
A list of symbol names.
"""
# TODO(lizeb,pasko): Move this method to symbol_extractor.py
for section in GetSectionsFromOrderfile(filename):
for symbol in _SectionNameToSymbols(RemoveSuffixes(section),
section_to_symbols_map):
yield symbol
def _SymbolsWithSameOffset(profiled_symbol, name_to_symbol_info,
offset_to_symbol_info):
"""Expands a symbol to include all symbols with the same offset.
Args:
profiled_symbol: the string symbol name to be expanded.
name_to_symbol_info: {name: [symbol_info1], ...}, as returned by
GetSymbolInfosFromBinary
offset_to_symbol_info: {offset: [symbol_info1, ...], ...}
Returns:
A list of symbol names, or an empty list if profiled_symbol was not in
name_to_symbol_info.
"""
if profiled_symbol not in name_to_symbol_info:
return []
symbol_infos = name_to_symbol_info[profiled_symbol]
expanded = []
for symbol_info in symbol_infos:
expanded += (s.name for s in offset_to_symbol_info[symbol_info.offset])
return expanded
@_UniqueGenerator
def _SectionMatchingRules(section_name, name_to_symbol_infos,
offset_to_symbol_infos, section_to_symbols_map,
symbol_to_sections_map, suffixed_sections):
"""Gets the set of section matching rules for section_name.
These rules will include section_name, but also any sections which may
contain the same code due to cloning, splitting, or identical code folding.
Args:
section_name: The section to expand.
name_to_symbol_infos: {name: [symbol_info1], ...}, as returned by
GetSymbolInfosFromBinary.
offset_to_symbol_infos: {offset: [symbol_info1, ...], ...}
section_to_symbols_map: The mapping from section to symbol name. Missing
section names are treated as per _SectionNameToSymbols.
symbol_to_sections_map: The mapping from symbol name to names of linker
sections containing the symbol. If a symbol isn't in the mapping, the
section names are generated from the set of _PREFIXES with the symbol
name.
suffixed_sections: A set of sections which can have suffixes.
Yields:
Section names including at least section_name.
"""
for name in _ExpandSection(section_name, name_to_symbol_infos,
offset_to_symbol_infos, section_to_symbols_map,
symbol_to_sections_map):
yield name
# Since only a subset of methods (mostly those compiled with O2) ever get
# suffixes, don't emit the wildcards for ones where it won't be helpful.
# Otherwise linking takes too long.
if name in suffixed_sections:
# TODO(lizeb,pasko): instead of just appending .*, append .suffix.* for
# _SUFFIXES. We can't do this right now because that many wildcards
# seems to kill the linker (linking libchrome takes 3 hours). This gets
# almost all the benefit at a much lower link-time cost, but could cause
# problems with unexpected suffixes.
yield name + '.*'
def _ExpandSection(section_name, name_to_symbol_infos, offset_to_symbol_infos,
section_to_symbols_map, symbol_to_sections_map):
"""Yields the set of section names for section_name.
This set will include section_name, but also any sections which may contain
the same code due to identical code folding.
Args:
section_name: The section to expand.
name_to_symbol_infos: {name: [symbol_info1], ...}, as returned by
GetSymbolInfosFromBinary.
offset_to_symbol_infos: {offset: [symbol_info1, ...], ...}
section_to_symbols_map: The mapping from section to symbol name. Missing
section names are treated as per _SectionNameToSymbols.
symbol_to_sections_map: The mapping from symbol name to names of linker
sections containing the symbol. If a symbol isn't in the mapping, the
section names are generated from the set of _PREFIXES with the symbol
name.
Yields:
Section names including at least section_name.
"""
yield section_name
for first_sym in _SectionNameToSymbols(section_name,
section_to_symbols_map):
for symbol in _SymbolsWithSameOffset(first_sym, name_to_symbol_infos,
offset_to_symbol_infos):
if symbol in symbol_to_sections_map:
for section in symbol_to_sections_map[symbol]:
yield section
for prefix in _PREFIXES:
yield prefix + symbol
@_UniqueGenerator
def _ExpandSections(section_names, name_to_symbol_infos,
offset_to_symbol_infos, section_to_symbols_map,
symbol_to_sections_map, suffixed_sections):
"""Gets an ordered set of section matching rules for a list of sections.
Rules will not be repeated.
Args:
section_names: The sections to expand.
name_to_symbol_infos: {name: [symbol_info1], ...}, as returned by
_GroupSymbolInfosFromBinary.
offset_to_symbol_infos: {offset: [symbol_info1, ...], ...}
section_to_symbols_map: The mapping from section to symbol names.
symbol_to_sections_map: The mapping from symbol name to names of linker
sections containing the symbol.
suffixed_sections: A set of sections which can have suffixes.
Yields:
Section matching rules including at least section_names.
"""
for profiled_section in section_names:
for section in _SectionMatchingRules(
profiled_section, name_to_symbol_infos, offset_to_symbol_infos,
section_to_symbols_map, symbol_to_sections_map, suffixed_sections):
yield section
def _CombineSectionListsByPrimaryName(symbol_to_sections_map):
"""Combines values of the symbol_to_sections_map by stripping suffixes.
Example:
{foo: [.text.foo, .text.bar.part.1],
foo.constprop.4: [.text.baz.constprop.3]} ->
{foo: [.text.foo, .text.bar, .text.baz]}
Args:
symbol_to_sections_map: Mapping from symbol name to list of section names
Returns:
The same mapping, but with symbol and section names suffix-stripped.
"""
simplified = {}
for suffixed_symbol, suffixed_sections in symbol_to_sections_map.iteritems():
symbol = RemoveSuffixes(suffixed_symbol)
sections = [RemoveSuffixes(section) for section in suffixed_sections]
simplified.setdefault(symbol, []).extend(sections)
return simplified
def _SectionsWithSuffixes(symbol_to_sections_map):
"""Finds sections which have suffixes applied.
Args:
symbol_to_sections_map: a map where the values are lists of section names.
Returns:
A set containing all section names which were seen with suffixes applied.
"""
sections_with_suffixes = set()
for suffixed_sections in symbol_to_sections_map.itervalues():
for suffixed_section in suffixed_sections:
section = RemoveSuffixes(suffixed_section)
if section != suffixed_section:
sections_with_suffixes.add(section)
return sections_with_suffixes
def _StripSuffixes(section_list):
"""Remove all suffixes on items in a list of sections or symbols."""
return [RemoveSuffixes(section) for section in section_list]
def GeneratePatchedOrderfile(unpatched_orderfile, native_lib_filename,
output_filename):
"""Writes a patched orderfile.
......@@ -364,20 +129,22 @@ def GeneratePatchedOrderfile(unpatched_orderfile, native_lib_filename,
native_lib_filename: (str) Path to the native library.
output_filename: (str) Path to the patched orderfile.
"""
(offset_to_symbol_infos, name_to_symbol_infos) = _GroupSymbolInfosFromBinary(
native_lib_filename)
obj_dir = cygprofile_utils.GetObjDir(native_lib_filename)
raw_symbol_map = cyglog_to_orderfile.ObjectFileProcessor(
obj_dir).GetSymbolToSectionsMap()
suffixed = _SectionsWithSuffixes(raw_symbol_map)
symbol_to_sections_map = _CombineSectionListsByPrimaryName(raw_symbol_map)
section_to_symbols_map = cygprofile_utils.InvertMapping(
symbol_to_sections_map)
profiled_sections = _StripSuffixes(
GetSectionsFromOrderfile(unpatched_orderfile))
expanded_sections = _ExpandSections(
profiled_sections, name_to_symbol_infos, offset_to_symbol_infos,
section_to_symbols_map, symbol_to_sections_map, suffixed)
symbol_to_matching = _GroupSymbolsByOffset(native_lib_filename)
profiled_symbols = ReadOrderfile(unpatched_orderfile)
missing_symbol_count = 0
seen_symbols = set()
patched_symbols = []
for sym in profiled_symbols:
if sym not in symbol_to_matching:
missing_symbol_count += 1
continue
if sym in seen_symbols:
continue
for matching in symbol_to_matching[sym]:
patched_symbols.append(matching)
seen_symbols.add(matching)
assert sym in seen_symbols
logging.warning('missing symbol count = %d', missing_symbol_count)
with open(output_filename, 'w') as f:
# Make sure the anchor functions are located in the right place, here and
......@@ -386,28 +153,14 @@ def GeneratePatchedOrderfile(unpatched_orderfile, native_lib_filename,
#
# __cxx_global_var_init is one of the largest symbols (~38kB as of May
# 2018), called extremely early, and not instrumented.
first_sections = ('dummy_function_start_of_ordered_text',
'__cxx_global_var_init')
for section in first_sections:
for prefix in _PREFIXES:
f.write(prefix + section + '\n')
for section in expanded_sections:
f.write(section + '\n')
for prefix in _PREFIXES:
f.write(prefix + 'dummy_function_end_of_ordered_text\n')
# The following is needed otherwise Gold only applies a partial sort.
f.write('.text\n') # gets methods not in a section, such as assembly
f.write('.text.*\n') # gets everything else
# Since wildcards are not supported by lld, the "end of text" anchor symbol
# is not emitted, a different mechanism is used instead. See comments in the
# file above.
for prefix in _PREFIXES:
if prefix:
f.write(prefix + 'dummy_function_at_the_end_of_text\n')
for first_section in ('dummy_function_start_of_ordered_text',
'__cxx_global_var_init'):
f.write(first_section + '\n')
for sym in patched_symbols:
f.write(sym + '\n')
f.write('dummy_function_end_of_ordered_text')
def _CreateArgumentParser():
......
tools/cygprofile/patch_orderfile_unittest.py
View file @ 8b141623
......@@ -20,80 +20,6 @@ class TestPatchOrderFile(unittest.TestCase):
self.assertEquals(
'this.is.a', patch_orderfile.RemoveSuffixes(with_part))
def testSymbolsWithSameOffset(self):
symbol_name = "dummySymbol"
symbol_name2 = "other"
name_to_symbol_infos = {symbol_name: [
symbol_extractor.SymbolInfo(symbol_name, 0x42, 0x12,
section='.text')]}
offset_to_symbol_infos = {
0x42: [symbol_extractor.SymbolInfo(symbol_name, 0x42, 0x12,
section='.text'),
symbol_extractor.SymbolInfo(symbol_name2, 0x42, 0x12,
section='.text')]}
symbol_names = patch_orderfile._SymbolsWithSameOffset(
symbol_name, name_to_symbol_infos, offset_to_symbol_infos)
self.assertEquals(len(symbol_names), 2)
self.assertEquals(symbol_names[0], symbol_name)
self.assertEquals(symbol_names[1], symbol_name2)
self.assertEquals([], patch_orderfile._SymbolsWithSameOffset(
"symbolThatShouldntMatch",
name_to_symbol_infos, offset_to_symbol_infos))
def testSectionNameToSymbols(self):
mapping = {'.text.foo': ['foo'],
'.text.hot.bar': ['bar', 'bar1']}
self.assertEquals(list(patch_orderfile._SectionNameToSymbols(
'.text.foo', mapping)),
['foo'])
self.assertEquals(list(patch_orderfile._SectionNameToSymbols(
'.text.hot.bar', mapping)),
['bar', 'bar1'])
self.assertEquals(list(patch_orderfile._SectionNameToSymbols(
'.text.hot.bar', mapping)),
['bar', 'bar1'])
self.assertEquals(list(patch_orderfile._SectionNameToSymbols(
'.text.hot.foobar', mapping)),
['foobar'])
self.assertEquals(list(patch_orderfile._SectionNameToSymbols(
'.text.unlikely.*', mapping)),
[])
def testSectionMatchingRules(self):
symbol_name1 = 'symbol1'
symbol_name2 = 'symbol2'
symbol_name3 = 'symbol3'
section_name1 = '.text.' + symbol_name1
section_name3 = '.text.foo'
suffixed = set([section_name3])
name_to_symbol_infos = {symbol_name1: [
symbol_extractor.SymbolInfo(symbol_name1, 0x42, 0x12,
section='.text')]}
offset_to_symbol_infos = {
0x42: [symbol_extractor.SymbolInfo(symbol_name1, 0x42, 0x12,
section='.text'),
symbol_extractor.SymbolInfo(symbol_name2, 0x42, 0x12,
section='.text')]}
section_to_symbols_map = {section_name1: [symbol_name1],
section_name3: [symbol_name1, symbol_name3]}
symbol_to_sections_map = {symbol_name1:
[section_name1, section_name3],
symbol_name3: [section_name3]}
expected = [
section_name1,
section_name3,
section_name3 + '.*',
'.text.hot.' + symbol_name1,
'.text.unlikely.' + symbol_name1,
symbol_name1,
'.text.hot.symbol2',
'.text.unlikely.symbol2',
'.text.symbol2',
'symbol2']
self.assertEqual(expected, list(patch_orderfile._SectionMatchingRules(
section_name1, name_to_symbol_infos, offset_to_symbol_infos,
section_to_symbols_map, symbol_to_sections_map, suffixed)))
def testUniqueGenerator(self):
@patch_orderfile._UniqueGenerator
def TestIterator():
......@@ -104,21 +30,6 @@ class TestPatchOrderFile(unittest.TestCase):
self.assertEqual(list(TestIterator()), [1,2,3])
def testCombineSectionListsByPrimaryName(self):
self.assertEqual(patch_orderfile._CombineSectionListsByPrimaryName(
{'foo': ['.text.foo', '.text.bar.constprop.1'],
'foo.part.1': ['.text.baz'],
'foobar': ['.text.foobar']}),
{'foo': ['.text.foo', '.text.bar', '.text.baz'],
'foobar': ['.text.foobar']})
def testSectionsWithSuffixes(self):
self.assertEqual(patch_orderfile._SectionsWithSuffixes(
{'foo': ['.text.foo', '.text.bar.constprop.1'],
'foo.part.1': ['.text.baz'],
'foobar': ['.text.foobar']}),
set(['.text.bar']))
if __name__ == "__main__":
unittest.main()
tools/cygprofile/process_profiles.py
View file @ 8b141623
......@@ -126,6 +126,30 @@ class SymbolOffsetProcessor(object):
self.SymbolInfos())
return self._offset_to_symbols
def GetOrderedSymbols(self, offsets):
"""Maps a list of offsets to symbol names, retaining ordering.
The symbol name is the primary symbol. This also deals with thumb
instruction (which have odd offsets).
Args::
offsets (int iterable) a set of offsets.
Returns
[str] list of symbol names.
"""
symbols = []
not_found = 0
for o in offsets:
if o in self.OffsetToPrimaryMap():
symbols.append(self.OffsetToPrimaryMap()[o].name)
elif o % 2 and (o - 1) in self.OffsetToPrimaryMap():
symbols.append(self.OffsetToPrimaryMap()[o - 1].name)
else:
not_found += 1
logging.warning('%d offsets do not have matching symbol', not_found)
return symbols
def OffsetsPrimarySize(self, offsets):
"""Computes the total primary size of a set of offsets.
......
tools/cygprofile/process_profiles_unittest.py
View file @ 8b141623
......@@ -58,6 +58,11 @@ class ProcessProfilesTestCase(unittest.TestCase):
self.assertDictEqual({8: symbol_infos[0],
40: symbol_infos[2]}, processor.OffsetToPrimaryMap())
def testGetOrderedSymbols(self):
processor = TestSymbolOffsetProcessor(self.symbol_infos)
self.assertListEqual(['1', '3', '0'],
processor.GetOrderedSymbols([8, 41, 6, 0]))
def testOffsetToSymbolsMap(self):
symbol_infos = [SimpleTestSymbol('1', 8, 16),
SimpleTestSymbol('AnAlias', 8, 16),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment