Improve sizeviewer for Windows

Now breaks down to the line level, and shows an annotated display
of the source file with the size cost of each line, as well as all
symbols that contribute to that size (can be many in the case of
templates).

Example output here:

https://dl.dropboxusercontent.com/u/116514/chrome.dll.html
https://dl.dropboxusercontent.com/u/116514/chrome_child.dll.html

Not really intended for loading remotely as they're very large,
but might eventually finish loading.

R=jam@chromium.org, mkosiba@chromium.org
BUG=441867

Review URL: https://codereview.chromium.org/801123002

Cr-Commit-Position: refs/heads/master@{#308394}

android_webview/tools/third_party_files_whitelist.txt

@@ -163,6 +163,10 @@ tools/symsrc/pefile.py
 tools/traceline/traceline/assembler.h
 # Copyright Google Inc; BSD license. Tool only.
 tools/traceline/traceline/sidestep/mini_disassembler.cc
+# Copyright Marijn Haverbeke. MIT license. Tool only, not used on Android.
+tools/win/sizeviewer/clike.js
+# Copyright Marijn Haverbeke. MIT license. Tool only, not used on Android.
+tools/win/sizeviewer/codemirror.js
 # Copyright The Chromium Authors, Apple Inc; BSD license. Not used on Android.
 ui/base/clipboard/clipboard_util_win.cc
 # Copyright The Chromium Authors, Apple Inc; BSD license. Not used on Android.

tools/win/sizeviewer/README.chromium

-code_tally.exe binary from
-http://syzygy-archive.commondatastorage.googleapis.com/index.html?path=builds/official/1626/
+code_tally.exe binary from
+http://syzygy-archive.commondatastorage.googleapis.com/index.html?path=builds/official/1626/
+
+favicon.png is chart_pie.png unmodified from
+http://www.famfamfam.com/lab/icons/silk/
+
+codemirror.js and clike.js are 4.8.0 unmodified from http://codemirror.net/.

tools/win/sizeviewer/main.js

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+google.load("visualization", "1", {packages:["treemap"]});
+google.setOnLoadCallback(drawChart);
+function drawChart() {
+  var data = google.visualization.arrayToDataTable(g_raw_data);
+
+  tree = new google.visualization.TreeMap(
+      document.getElementById('chart_div'));
+
+  tree.draw(data, {
+    minColor: '#faa',
+    midColor: '#f77',
+    maxColor: '#f44',
+    headerHeight: 20,
+    fontColor: 'black',
+    showScale: true,
+    minColorValue: 0,
+    maxColorValue: g_maxval,
+    generateTooltip: tooltip
+  });
+
+  // Update from 'Loading'.
+  document.getElementById('title').innerText = g_dllname;
+
+  // Set favicon.
+  var doc_head = document.getElementsByTagName('head')[0];
+  var new_link = document.createElement('link');
+  new_link.rel = 'shortcut icon';
+  new_link.href = 'data:image/png;base64,'+g_favicon;
+  doc_head.appendChild(new_link);
+
+  var cur_line_sizes = null;
+  function nodeSelect() {
+    symlist.setValue('');
+    var selected = tree.getSelection();
+    if (selected.length > 0) {
+      var filename = data.getValue(selected[0].row, 0);
+      var size = data.getValue(selected[0].row, 2);
+      if (size >= 0) {
+        // Is a leaf.
+        cur_line_sizes = g_line_data[filename];
+        var body = g_file_contents[filename];
+        editor.setValue(body);
+        var maximum_size = 0;
+        for (var line in cur_line_sizes) {
+          maximum_size = Math.max(maximum_size, cur_line_sizes[line][0]);
+        }
+        for (var line in cur_line_sizes) {
+          var symbol_indices = cur_line_sizes[line][1];
+          var symbols = [];
+          for (var i = 0; i < symbol_indices.length; ++i) {
+            symbols.push(g_symbol_list[symbol_indices[i]]);
+          }
+          var size = cur_line_sizes[line][0];
+          // Zero based lines.
+          var line_num = parseInt(line, 10) - 1;
+          if (size >= maximum_size * 0.9)
+            editor.addLineClass(line_num, 'gutter', 'linebg-top10');
+          else if (size >= maximum_size * 0.75)
+            editor.addLineClass(line_num, 'gutter', 'linebg-top25');
+          else if (size >= maximum_size * 0.5)
+            editor.addLineClass(line_num, 'gutter', 'linebg-top50');
+          function addTag() {
+            var line_num = parseInt(line, 10);
+            var symbols_tooltip = symbols.join('\n');
+            var num_syms = symbols.length;
+            // markText wants 0-based lines.
+            var mark = editor.markText({line: line_num - 1, ch: 0},
+                            {line: line_num, ch: 0},
+                            { className: 'symbol-tag' });
+            CodeMirror.on(mark, 'beforeCursorEnter', function(e) {
+              symlist.setValue(num_syms +
+                                ' symbol(s) contributing to line ' +
+                                line_num + ':\n' +
+                                symbols_tooltip);
+            });
+          }
+          addTag();
+        }
+      }
+    }
+  }
+  google.visualization.events.addListener(tree, 'select', nodeSelect);
+
+  editor = CodeMirror.fromTextArea(
+    document.getElementById('source_view'), {
+      readOnly: "nocursor",
+      mode: { name: 'text/x-c++src' },
+      lineNumbers: true,
+      lineNumberFormatter: weightGetter
+    });
+  editor.setSize(850, 600);
+
+  symlist = CodeMirror.fromTextArea(
+    document.getElementById('symlist_view'), {
+      readOnly: "nocursor",
+      mode: { name: 'none' },
+      lineNumbers: false
+    });
+  symlist.setSize(850, 150);
+
+  function tooltip(row, size, value) {
+    return '<div style="background:#fd9;' +
+            ' padding:10px; border-style:solid"><b>' +
+            data.getValue(row, 0) + '</b><br>' +
+            data.getColumnLabel(2) +
+            ' (total value of this cell and its children): ' + size +
+            '<br>';
+  }
+
+  function weightGetter(line) {
+    if (cur_line_sizes && cur_line_sizes.hasOwnProperty('' + line)) {
+      return cur_line_sizes['' + line][0] + ' bytes ' + line;
+    }
+    return line;
+  }
+}

tools/win/sizeviewer/sizeviewer.py

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import base64
+import codecs
 import json
 import os
 import string
@@ -25,12 +27,12 @@ def FindNode(node, component):
 
 
 def InsertIntoTree(tree, source_name, size):
-  components = source_name.replace(':', '').split('\\')
+  components = source_name[3:].split('\\')
   node = tree
   for index, component in enumerate(components):
     data = FindNode(node, component)
     if not data:
-      data = { 'name': component }
+      data = { 'name': source_name, 'name': component }
       if index == len(components) - 1:
         data['size'] = size
       else:
@@ -39,6 +41,37 @@ def InsertIntoTree(tree, source_name, size):
     node = data
 
 
+def FlattenTree(tree):
+  result = [['Path', 'Parent', 'Size', 'Value']]
+  def Flatten(node, parent):
+    name = node['name']
+    if parent and parent != '/':
+      name = parent + '/' + name
+    if 'children' in node:
+      result.append([name, parent, -1, -1])
+      for c in node['children']:
+        Flatten(c, name)
+    else:
+      result.append([name, parent, node['size'], node['size']])
+  Flatten(tree, '')
+  return result
+
+
+def GetAsset(filename):
+  with open(os.path.join(BASE_DIR, filename), 'rb') as f:
+    return f.read()
+
+
+def AppendAsScriptBlock(f, value, var=None):
+  f.write('<script type="text/javascript">\n')
+  if var:
+    f.write('var ' + var + ' = ')
+  f.write(value)
+  if var:
+    f.write(';\n')
+  f.write('</script>\n')
+
+
 def main():
   out_dir = os.path.join(BASE_DIR, '..', '..', '..', 'out', 'Release')
   jsons = []
@@ -56,37 +89,99 @@ def main():
     print 'Couldn\'t find binaries, looking in', out_dir
     return 1
 
+  # Munge the code_tally json format into an easier-to-view format.
   for json_name in jsons:
     with open(json_name, 'r') as jsonf:
       all_data = json.load(jsonf)
     html_path = os.path.splitext(json_name)[0] + '.html'
-    print 'Generating %s...' % html_path
+    print 'Generating %s... (standlone)' % html_path
     by_source = {}
+    symbols_index = {}
+    symbols = []
     for obj_name, obj_data in all_data['objects'].iteritems():
       for symbol, symbol_data in obj_data.iteritems():
         size = int(symbol_data['size'])
         # Sometimes there's symbols with no source file, we just ignore those.
         if 'contribs' in symbol_data:
-          # There may be more than one file in the list, we just assign to the
-          # first source file that contains the symbol, rather than try to
-          # split or duplicate info.
-          src_index = symbol_data['contribs'][0]
-          source = all_data['sources'][int(src_index)]
-          if source not in by_source:
-            by_source[source] = []
-          by_source[source].append(size)
+          i = 0
+          while i < len(symbol_data['contribs']):
+            src_index = symbol_data['contribs'][i]
+            i += 1
+            per_line = symbol_data['contribs'][i]
+            i += 1
+            source = all_data['sources'][int(src_index)]
+            if source not in by_source:
+              by_source[source] = {'lines': {}, 'total_size': 0}
+            size = 0
+            # per_line is [line, size, line, size, line, size, ...]
+            for j in range(0, len(per_line), 2):
+              line_number = per_line[j]
+              size += per_line[j + 1]
+              # Save some time/space in JS by using an array here. 0 == size,
+              # 1 == symbol list.
+              by_source[source]['lines'].setdefault(line_number, [0, []])
+              by_source[source]['lines'][line_number][0] += per_line[j + 1]
+              if symbol in symbols_index:
+                symindex = symbols_index[symbol]
+              else:
+                symbols.append(symbol)
+                symbols_index[symbol] = symindex = len(symbols) - 1
+              by_source[source]['lines'][line_number][1].append(
+                  symindex)
+            by_source[source]['total_size'] += size
     binary_name = all_data['executable']['name']
     data = {}
-    data['name'] = binary_name
+    data['name'] = '/'
     data['children'] = []
-    for source, sizes in by_source.iteritems():
-      InsertIntoTree(data, source, sum(sizes))
+    file_contents = {}
+    line_data = {}
+    for source, file_data in by_source.iteritems():
+      InsertIntoTree(data, source, file_data['total_size'])
+
+      store_as = source[3:].replace('\\', '/')
+      try:
+        with codecs.open(source, 'rb', encoding='latin1') as f:
+          file_contents[store_as] = f.read()
+      except IOError:
+        file_contents[store_as] = '// Unable to load source.'
+
+      line_data[store_as] = file_data['lines']
+      # code_tally attempts to assign fractional bytes when code is shared
+      # across multiple symbols. Round off here for display after summing above.
+      for per_line in line_data[store_as].values():
+        per_line[0] = round(per_line[0])
+
+    flattened = FlattenTree(data)
+    maxval = 0
+    for i in flattened[1:]:
+      maxval = max(i[2], maxval)
+    flattened_str = json.dumps(flattened)
+
+    to_write = GetAsset('template.html')
+    # Save all data and what would normally be external resources into the
+    # one html so that it's a standalone report.
     with open(html_path, 'w') as f:
-      with open(os.path.join(BASE_DIR, 'template.html'), 'r') as templatef:
-        template = templatef.read()
-      f.write(string.Template(template).substitute(
-          {'data': json.dumps(data, indent=2),
-           'dllname': binary_name + ' ' + all_data['executable']['version']}))
+      f.write(to_write)
+      # These aren't subbed in as a silly workaround for 32-bit python.
+      # The end result is only ~100M, but while substituting these into a
+      # template, it otherwise raises a MemoryError, I guess due to
+      # fragmentation. So instead, we just append them as variables to the file
+      # and then refer to the variables in the main script.
+      filedata_str = json.dumps(file_contents).replace(
+          '</script>', '</scr"+"ipt>')
+      AppendAsScriptBlock(f, filedata_str, var='g_file_contents')
+      AppendAsScriptBlock(f, json.dumps(line_data), var='g_line_data')
+      AppendAsScriptBlock(f, json.dumps(symbols), var='g_symbol_list')
+      favicon_str = json.dumps(base64.b64encode(GetAsset('favicon.png')))
+      AppendAsScriptBlock(f, favicon_str, var='g_favicon')
+      AppendAsScriptBlock(f, flattened_str, var='g_raw_data')
+      AppendAsScriptBlock(f, str(maxval), var='g_maxval')
+      dllname_str = binary_name + ' ' + all_data['executable']['version']
+      AppendAsScriptBlock(f, json.dumps(dllname_str), var='g_dllname')
+      AppendAsScriptBlock(f, GetAsset('codemirror.js'))
+      AppendAsScriptBlock(f, GetAsset('clike.js'))
+      AppendAsScriptBlock(f, GetAsset('main.js'))
+      f.write('</html>')
 
   return 0