Add tool to help analyze binary size

BUG=

Review URL: https://codereview.chromium.org/119083006

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@245405 0039d316-1c4b-4281-b951-d872f2087c98

build/all_android.gyp

@@ -183,5 +183,13 @@
       'dependencies': [
       ],
     },
+    {
+      # Build the java portions of the binary size analysis tool.
+      'target_name': 'binary_size_tool',
+      'type': 'none',
+      'dependencies': [
+        '../tools/binary_size/binary_size.gyp:binary_size_java',
+      ],
+    },
   ],  # targets
 }

tools/binary_size/README.txt

+================================================================================
+              __________  .__
+              \______   \ |__|   ____   _____    _______   ___.__.
+               |    |  _/ |  |  /    \  \__  \   \_  __ \ <   |  |
+               |    |   \ |  | |   |  \  / __ \_  |  | \/  \___  |
+               |______  / |__| |___|  / (____  /  |__|     / ____|
+                      \/            \/       \/            \/
+    _________ .__                        ___________                   .__
+   /   _____/ |__| ________   ____       \__    ___/   ____     ____   |  |
+   \_____  \  |  | \___   / _/ __ \        |    |     /  _ \   /  _ \  |  |
+   /        \ |  |  /    /  \  ___/        |    |    (  <_> ) (  <_> ) |  |__
+  /_______  / |__| /_____ \  \___  >       |____|     \____/   \____/  |____/
+          \/             \/      \/
+================================================================================
+
+--------------------------------------------------------------------------------
+Introduction
+--------------------------------------------------------------------------------
+The ever-increasing size of binaries is a problem for everybody. Increased
+binary size means longer download times and a bigger on-disk footprint after
+installation. Mobile devices suffer the worst, as they frequently have
+sub-optimal connectivity and limited storage capacity. Developers currently
+have almost no visibility into how the space in the existing binaries is
+divided nor how their contributions change the space within those binaries.
+The first step to reducing the size of binaries is to make the size information
+accessible to everyone so that developers can take action.
+
+The Binary Size Tool does the following:
+  1. Runs "nm" on a specified binary to dump the symbol table
+  2. Runs a parallel pool of "addr2line" processes to map the symbols from the
+     symbol table back to source code (way faster than running "nm -l")
+  3. Creates (and optionally saves) an intermediate file that accurately mimcs
+     (binary compatible with) the "nm" output format, with all the source code
+     mappings present
+  4. Parses, sorts and analyzes the results
+  5. Generates an HTML-based report in the specified output directory
+
+
+--------------------------------------------------------------------------------
+How to Run
+--------------------------------------------------------------------------------
+Running the tool is fairly simply. For the sake of this example we will
+pretend that you are building the Content Shell APK for Android.
+
+  1. Build your product as you normally would, e.g.:
+       ninja -C out/Release -j 100 content_shell_apk
+
+  2. Build the "binary_size_tool" target from ../../build/all_android.gyp, e.g.:
+       ninja -C out/Release binary_size_tool
+
+  3. Run the tool specifying the library and the output report directory.
+     This command will run the analysis on the Content Shell native library for
+     Android using the nm/addr2line binaries from the Android NDK for ARM,
+     producing an HTMl report in /tmp/report:
+       tools/binary_size/run_binary_size_analysis.py \
+         --library out/Release/lib/libcontent_shell_content_view.so \
+         --destdir /tmp/report \
+         --arch android-arm
+
+Of course, there are additional options that you can see by running the tool
+with "--help".
+
+This whole process takes about an hour on a modern (circa 2014) quad-core
+machine. If you have LOTS of RAM, you can use the "--jobs" argument to
+add more addr2line workers; doing so will *greatly* reduce the processing time
+but will devour system memory. If you've got the horsepower, 10 workers can
+thrash through the binary in about 5 minutes at a cost of around 60 GB of RAM.
+
+--------------------------------------------------------------------------------
+Analyzing the Output
+--------------------------------------------------------------------------------
+When the tool finishes its work you'll find an HTML report in the output
+directory that you specified with "--destdir". Open the index.html file in your
+*cough* browser of choice *cough* and have a look around. The index.html page
+is likely to evolve over time, but will always be your starting point for
+investigation. From here you'll find links to various views of the data such
+as treemap visualizations, overall statistics and "top n" lists of various
+kinds.
+
+The report is completely standalone. No external resources are required, so the
+report may be saved and viewed offline with no problems.
+
+--------------------------------------------------------------------------------
+Caveats
+--------------------------------------------------------------------------------
+The tool is not perfect and has several shortcomings:
+
+  * Not all space in the binary is accounted for. The cause is still under
+    investigation.
+  * The tool is partly written in Java, temporarily tying it to Android
+    purely and solely because it needs Java build logic which is only defined
+    in the Android part of the build system. The Java portions need to be
+    rewritten in Python so we can decouple from Android, or we need to find
+    an alternative (readelf, linker maps, etc) to running nm/addr2line.
+  * The Java code assumes that the library file is within a Chromium release
+    directory. This limits it to Chromium-based binaries only.
+  * The Java code has a hack to accurately identify the source of ICU data
+    within the Chromium source tree due to missing symbols in the ICU ASM
+    output.
+  * The Python script assumes that arm-based and mips-based nm/addr2line
+    binaries exist in ../../third_party. This is true only when dealing with
+    Android and again limits the tool to Chromium-based binaries.
+  * The Python script uses build system variables to construct the classpath
+    for running the Java code.
+  * The Javascript code in the HTML report Assumes code lives in Chromium for
+    generated hyperlinks and will not hyperlink any file that starts with the
+    substring "out".
+
+--------------------------------------------------------------------------------
+Feature Requests and Bug Reports
+--------------------------------------------------------------------------------
+Please file bugs and feature requests here, making sure to use the label
+"Binary-Size-Tool":
+  https://code.google.com/p/chromium/issues/entry?labels=Binary-Size-Tool
+
+View all open issues here:
+  https://code.google.com/p/chromium/issues/list?can=2&q=label:Binary-Size-Tool
\ No newline at end of file

tools/binary_size/binary_size.gyp

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+{
+  'targets': [
+    {
+      'target_name': 'binary_size_java',
+      'type': 'none',
+      'variables': {
+        'java_in_dir': '../binary_size/java',
+      },
+      'includes': [ '../../build/java.gypi' ],
+    },
+  ],
+}

tools/binary_size/java/src/org/chromium/tools/binary_size/NmDumper.java

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+package org.chromium.tools.binary_size;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Converts records to a format that simulates output from 'nm'.
+ */
+class NmDumper {
+    private final String mOutPath;
+    private final String mSkipPath;
+    private final String mFailPath;
+    private final Output mOutput;
+
+    /**
+     * Create a new dumper that will output to the specified paths.
+     * @param outPath where to write all records and lines, including lines
+     * that are skipped or records that failed to resolve
+     * @param failPath if not null, a path to which <em>only</em> records that
+     * failed to resolve will be written
+     * @param skipPath if not null, a path to which <em>only</em> lines that
+     * were skipped will be written
+     */
+    NmDumper(final String outPath, final String failPath, final String skipPath) {
+        mOutPath = outPath;
+        mFailPath = failPath;
+        mSkipPath = skipPath;
+        mOutput = new Output();
+    }
+
+    /**
+     * Close all output files.
+     */
+    void close() {
+        mOutput.closeAll();
+    }
+
+    /**
+     * Output a line that was skipped.
+     * @param line the line
+     */
+    void skipped(String line) {
+        mOutput.printSkip(line);
+    }
+
+    /**
+     * Output a record that failed to resolve.
+     * @param record the record
+     */
+    void failed(Record record) {
+        mOutput.printFail(simulateNmOutput(record));
+    }
+
+    /**
+     * Output a record that successfully resolved.
+     * @param record the record
+     */
+    void succeeded(Record record) {
+        mOutput.print(simulateNmOutput(record));
+    }
+
+    /**
+     * Generate a string that looks like output from nm for a given record.
+     * @param record the record
+     * @return nm-like output
+     */
+    private static final String simulateNmOutput(final Record record) {
+        StringBuilder builder = new StringBuilder(record.address);
+        builder.append(' ');
+        builder.append(record.size);
+        builder.append(' ');
+        builder.append(record.symbolType);
+        builder.append(' ');
+        builder.append(record.symbolName != null ? record.symbolName : "unknown");
+        if (record.location != null) {
+            builder.append('\t');
+            builder.append(record.location);
+        }
+        return builder.toString();
+    }
+
+    private class Output {
+        private final PrintWriter skipWriter;
+        private final PrintWriter failWriter;
+        private final PrintWriter outWriter;
+
+        private Output() {
+            try {
+                new File(mOutPath).getParentFile().mkdirs();
+                outWriter = new PrintWriter(mOutPath);
+            } catch (FileNotFoundException e) {
+                throw new RuntimeException("Can't open output file: " + mOutPath, e);
+            }
+
+            if (mFailPath != null) {
+                try {
+                    new File(mFailPath).getParentFile().mkdirs();
+                    failWriter = new PrintWriter(mFailPath);
+                } catch (FileNotFoundException e) {
+                    throw new RuntimeException("Can't open fail file: " + mFailPath, e);
+                }
+            } else {
+                failWriter = null;
+            }
+
+            if (mSkipPath != null) {
+                try {
+                    new File(mSkipPath).getParentFile().mkdirs();
+                    skipWriter = new PrintWriter(mSkipPath);
+                } catch (IOException e) {
+                    throw new RuntimeException("Can't open skip file: " + mSkipPath, e);
+                }
+            } else  {
+                skipWriter = null;
+            }
+        }
+
+        private synchronized void println(PrintWriter writer, String string) {
+            if (writer != null) {
+                writer.println(string);
+                writer.flush();
+            }
+        }
+
+        synchronized void print(String string) {
+            println(outWriter, string);
+        }
+
+        synchronized void printSkip(String string) {
+            println(skipWriter, string);
+            println(outWriter, string);
+        }
+
+        synchronized void printFail(String string) {
+            println(failWriter, string);
+            println(outWriter, string);
+        }
+
+        private void closeAll() {
+            for (PrintWriter writer : new PrintWriter[]{outWriter, failWriter, skipWriter}) {
+                if (writer != null) {
+                    try {
+                        writer.flush(); }
+                    catch (Exception ignored) {
+                        // Nothing to be done.
+                    }
+                    try {
+                        writer.close(); }
+                    catch (Exception ignored) {
+                        // Nothing to be done.
+                    }
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

tools/binary_size/java/src/org/chromium/tools/binary_size/Record.java

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+package org.chromium.tools.binary_size;
+
+/**
+ * A record that is filled in partially by nm and partially by addr2line,
+ * along with tracking information about whether or not the lookup in
+ * addr2line was successful.
+ */
+class Record {
+    /**
+     * The base-16 address, as a string.
+     */
+    String address;
+
+    /**
+     * The symbol type.
+     */
+    char symbolType;
+
+    /**
+     * The name of the symbol. Note that this may include whitespace, but
+     * not tabs.
+     */
+    String symbolName;
+
+    /**
+     * The base-10 size in bytes, as a String.
+     */
+    String size;
+
+    /**
+     * The location, if available; may include a file name and, optionally,
+     * a colon separator character followed by a line number or a
+     * question mark.
+     */
+    String location;
+
+    /**
+     * Whether or not the record was successfully resolved. Records that are
+     * successfully resolved should have a non-null location.
+     */
+    boolean resolvedSuccessfully;
+}
\ No newline at end of file

tools/binary_size/template/.gitignore

+*.json

tools/binary_size/template/index.html

+<!DOCTYPE html>
+<!--
+  Copyright 2014 The Chromium Authors. All rights reserved.
+  Use of this source code is governed by a BSD-style license that can be
+  found in the LICENSE file.
+-->
+<html>
+<head>
+  <title>Binary Size Analysis</title>
+  <link rel='stylesheet' href='webtreemap/webtreemap.css'>
+  <style>
+    body { font-family: sans-serif; }
+    tt, pre { font-family: WebKitWorkaround, monospace; }
+    #map {
+      margin: 0 auto;
+      position: relative;
+      cursor: pointer;
+      -webkit-user-select: none;
+    }
+    #table {
+      border: 1px solid black;
+    }
+    .treemaplegend {
+      margin: 0 auto;
+      position: relative;
+    }
+    .webtreemap-symbol-vtable {
+      background: #FFFFAA;
+    }
+    .webtreemap-node:hover {
+      border-color: red;
+      background: linear-gradient(rgb(240,240,200), rgb(180,180,200));
+    }
+  </style>
+  <script src='webtreemap/webtreemap.js'></script>
+  <script src='treemap-dump.js'></script>
+  <script src='largest-symbols.js'></script>
+  <script src='largest-sources.js'></script>
+  <script src='largest-vtables.js'></script>
+</head>
+<body onload='show_report_treemap()'>
+<div style='text-align: center; margin-bottom: 2em;'>
+  <h1>Binary Size Analysis</h1>
+  <a href='#' onclick='show_report_treemap()'>Spatial Treemap</a>
+  ~
+  <a href='#' onclick='show_report_largest_sources()'>Largest Sources</a>
+  ~
+  <a href='#' onclick='show_report_largest_symbols()'>Largest Symbols</a>
+  ~
+  <a href='#' onclick='show_report_largest_vtables()'>Largest VTables</a>
+</div>
+<div id='report'></div>
+<script>
+function escape(str) { 
+  return str.replace(/&/g, '&amp;')
+            .replace(/"/g, '&quot;')
+            .replace(/</g, '&lt;')
+            .replace(/>/g, '&gt;');
+}
+
+var treemap_width = 800;
+var treemap_height = 450;
+function show_report_treemap() {
+  console.log('displaying treemap')
+  var div = document.getElementById('report');
+  var w = window.treemap_width;
+  var h = window.treemap_height;
+  div.innerHTML = '<div style=\'text-align: center;\'>' +
+                  '<button onclick=\'zoomInTreemap()\'>Zoom In</button>' +
+                  ', <button onclick=\'zoomOutTreemap()\'>Zoom Out</button>' +
+                  ' or resize to: ' +
+                  '<input type=text size=5 id=treemap_width value=' + w + '>x' +
+                  '<input type=text size=5 id=treemap_height value=' + h + '>' +
+                  '<button onclick=\'resizeTreemap()\'>Go</button>' +
+                  '<br><em>Click on a box to zoom in. ' +
+                  'Click on the outermost box to zoom out.</em>' +
+                  '<br>Legend: <table border=1 class=\'treemaplegend\' cellborder=1><tr>' +
+                  '<td class=\'webtreemap-symbol-bss\'>BSS</td>' +
+                  '<td class=\'webtreemap-symbol-data\'>Data</td>' +
+                  '<td class=\'webtreemap-symbol-code\'>Code</td>' +
+                  '<td class=\'webtreemap-symbol-read-only_data\'>RO Data</td>' +
+                  '<td class=\'webtreemap-symbol-weak_symbol\'>Weak</td>' +
+                  '<td class=\'webtreemap-symbol-vtable\'>VTable</td>' +
+                  '</tr></table>' +
+                  '<br>' +
+                  '<div id=\'map\' ' +
+                  'style=\'width: ' + w + 'px; height: ' + h + 'px;\'>' +
+                  '</div></div>';
+  var map = document.getElementById('map');
+  appendTreemap(map, kTree);
+}
+
+function zoomInTreemap() {
+  window.treemap_width = Math.round(window.treemap_width * 1.25);
+  window.treemap_height = Math.round(window.treemap_height * 1.25);
+  show_report_treemap();
+}
+
+function zoomOutTreemap() {
+  window.treemap_width = Math.round(window.treemap_width / 1.25);
+  window.treemap_height = Math.round(window.treemap_height / 1.25);
+  show_report_treemap();
+}
+
+function resizeTreemap() {
+  window.treemap_width = document.getElementById('treemap_width').value;
+  window.treemap_height = document.getElementById('treemap_height').value;
+  show_report_treemap();
+}
+
+function show_report_largest_symbols() {
+  console.log('displaying largest-symbols report')
+  var div = document.getElementById('report');
+  div.innerHTML = '<div><table id=\'list\' border=1><tr>' +
+                  '<th>Rank</th><th>Size</th><th>Type</th><th>Source</th>' +
+                  '</tr></table>';
+  var list = document.getElementById('list');
+  for (var i = 0; i < largestSymbols.length; i++) {
+    var record = largestSymbols[i];
+    var link;
+    if (record.location.indexOf('out') == 0) {
+      link = record.location;
+    } else {
+      link = '<a href="https://code.google.com/p/chromium/codesearch#chromium/src/'
+             + record.location + '">' + escape(record.location) + '</a>';
+    }
+    list.innerHTML += '<tr>'
+      + '<td>' + (i+1) + '</td>'
+      + '<td>' + escape(record.size) + '</td>'
+      + '<td style=\'white-space: nowrap;\'>' + escape(record.type) + '</td>'
+      + '<td>' + link + ':<br>'
+      + escape(record.symbol) + '</td>'
+      + '</tr>';
+  }
+}
+
+function show_report_largest_sources() {
+  console.log('displaying largest-sources report')
+  var div = document.getElementById('report');
+  div.innerHTML = '<div><table id=\'list\' border=1><tr>' +
+                  '<th>Rank</th><th>Size</th><th>Symbol Count</th><th>Source</th>' +
+                  '</tr></table>';
+  var list = document.getElementById('list');
+  for (var i = 0; i < largestSources.length; i++) {
+    var record = largestSources[i];
+    var link;
+    if (record.location.indexOf('out') == 0) {
+      link = record.location;
+    } else {
+      link = '<a href="https://code.google.com/p/chromium/codesearch#chromium/src/'
+             + record.location + '">' + escape(record.location) + '</a>';
+    }
+
+    list.innerHTML += '<tr>'
+      + '<td>' + (i+1) + '</td>'
+      + '<td>' + escape(record.size) + '</td>'
+      + '<td>' + escape(record.symbol_count) + '</td>'
+      + '<td>' + link + '</td>'
+      + '</tr>';
+  }
+}
+
+function show_report_largest_vtables() {
+  console.log('displaying largest-vtables report')
+  var div = document.getElementById('report');
+  div.innerHTML = '<div><table id=\'list\' border=1><tr>' +
+                  '<th>Rank</th><th>Size</th><th>Symbol</th><th>Source</th>' +
+                  '</tr></table>';
+  var list = document.getElementById('list');
+  for (var i = 0; i < largestVTables.length; i++) {
+    var record = largestVTables[i];
+    var link;
+    if (record.location.indexOf('out') == 0) {
+      link = record.location;
+    } else {
+      link = '<a href="https://code.google.com/p/chromium/codesearch#chromium/src/'
+             + record.location + '">' + escape(record.location) + '</a>';
+    }
+
+    list.innerHTML += '<tr>'
+      + '<td>' + (i+1) + '</td>'
+      + '<td>' + escape(record.size) + '</td>'
+      + '<td>' + escape(record.symbol) + '</td>'
+      + '<td>' + link + '</td>'
+      + '</tr>';
+  }
+}
+</script>
+</body>
+</html>
\ No newline at end of file