SuperSize: Construct file tree in WebAssembly

This establishes the framework through which the WebAssembly-backed web
worker will send tree updates to the viewer thread.

Currently WebAssembly constructs a JSON object, writes it to a char* in
heap memory, passes that pointer to JavaScript, which reads the string
out of heap memory, parses it into a JS object, and frees the heap
pointer. Since sending the JS object from the web worker to the viewer
thread involves a serialization/deserialization, this is performing an
extra serialization/deserialization, but because any given tree load is
only loading a 1-deep view of the file tree, the performance hit is very
small. This sacrifice lets us maintain a common web worker API between
the wasm and non-wasm implementations.

TODOs have been liberally sprinkled through the CL reflecting future
functionality like node filtering, components, and type-based size
breakdowns.

Bug: 1011921
Change-Id: I383ec97bf252657c8840d0e36cb99f52645116c9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1873108Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#709479}

tools/binary_size/libsupersize/caspian/caspian_web.cc

@@ -7,14 +7,174 @@
 
 #include <stdlib.h>
 
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+
+#include "third_party/jsoncpp/source/include/json/json.h"
 #include "tools/binary_size/libsupersize/caspian/file_format.h"
 #include "tools/binary_size/libsupersize/caspian/model.h"
 
+namespace {
 caspian::SizeInfo info;
 
+caspian::TreeNode root;
+// TODO: A full hash table might be overkill here - could walk tree to find
+// node.
+std::unordered_map<std::string_view, caspian::TreeNode*> parents;
+
+std::unique_ptr<Json::StreamWriter> writer;
+
+/** Name used by a directory created to hold symbols with no name. */
+const char* _NO_NAME = "(No path)";
+
+std::string ToJson(const Json::Value& value) {
+  std::stringstream s;
+  writer->write(value, &s);
+  return s.str();
+}
+
+void OpenIntoJson(const caspian::TreeNode* node, Json::Value* out, int depth) {
+  (*out)["idPath"] = std::string(node->id_path);
+  (*out)["shortNameIndex"] = node->short_name_index;
+  // TODO: Put correct information.
+  (*out)["type"] = "D";
+  (*out)["size"] = node->size;
+  (*out)["flags"] = node->flags;
+  (*out)["childStats"] = Json::Value(Json::objectValue);
+  if (depth < 0) {
+    (*out)["children"] = Json::Value();  // null
+  } else {
+    (*out)["children"] = Json::Value(Json::arrayValue);
+    for (unsigned int i = 0; i < node->children.size(); i++) {
+      OpenIntoJson(node->children[i], &(*out)["children"][i], depth - 1);
+    }
+  }
+}
+
+std::string_view DirName(std::string_view path, char sep) {
+  auto end = path.find_last_of(sep);
+  if (end != std::string_view::npos) {
+    std::string_view ret = path;
+    ret.remove_suffix(path.size() - end);
+    return ret;
+  }
+  return "";
+}
+
+void AttachToParent(caspian::TreeNode* child, caspian::TreeNode* parent) {
+  if (child->parent != nullptr) {
+    std::cerr << "Child " << child->id_path << " already attached to parent "
+              << child->parent->id_path << std::endl;
+    std::cerr << "Cannot be attached to " << parent->id_path << std::endl;
+    exit(1);
+  }
+
+  parent->children.push_back(child);
+  child->parent = parent;
+
+  // Update size information along tree.
+  caspian::TreeNode* node = child;
+  while (node->parent) {
+    node->parent->size += child->size;
+    node = node->parent;
+  }
+}
+
+caspian::TreeNode* GetOrMakeParentNode(caspian::TreeNode* child_node) {
+  std::string_view parent_path;
+  if (child_node->id_path.empty()) {
+    parent_path = _NO_NAME;
+  } else {
+    parent_path = DirName(child_node->id_path, '/');
+  }
+
+  caspian::TreeNode*& parent = parents[parent_path];
+  if (parent == nullptr) {
+    parent = new caspian::TreeNode();
+    parent->id_path = parent_path;
+    parent->short_name_index = parent_path.find_last_of('/') + 1;
+    // TODO: Set parent type to directory or component.
+    parents[parent_path] = parent;
+  }
+  if (child_node->parent != parent) {
+    AttachToParent(child_node, parent);
+  }
+  return parent;
+}
+
+void AddFileEntry(const std::string_view source_path,
+                  const std::vector<const caspian::Symbol*>& symbols) {
+  // Creates a single file node with a child for each symbol in that file.
+  caspian::TreeNode* file_node = new caspian::TreeNode();
+  if (source_path.empty()) {
+    file_node->id_path = _NO_NAME;
+  } else {
+    file_node->id_path = source_path;
+  }
+  file_node->short_name_index = source_path.find_last_of('/') + 1;
+  // TODO: Initialize file type, source path, component
+
+  // Create symbol nodes.
+  for (const auto sym : symbols) {
+    caspian::TreeNode* symbol_node = new caspian::TreeNode();
+    symbol_node->id_path = sym->full_name;
+    symbol_node->size = sym->size;
+    AttachToParent(symbol_node, file_node);
+  }
+
+  // TODO: Only add if there are unfiltered symbols in this file.
+  caspian::TreeNode* orphan_node = file_node;
+  while (orphan_node != &root) {
+    orphan_node = GetOrMakeParentNode(orphan_node);
+  }
+}
+
+void AddSymbolsAndFileNodes(caspian::SizeInfo* size_info) {
+  // Group symbols by source path.
+  std::unordered_map<std::string_view, std::vector<const caspian::Symbol*>>
+      symbols;
+  for (auto& sym : size_info->raw_symbols) {
+    std::string_view key = sym.source_path;
+    if (key == nullptr) {
+      key = sym.object_path;
+    }
+    symbols[key].push_back(&sym);
+  }
+  for (const auto& pair : symbols) {
+    AddFileEntry(pair.first, pair.second);
+  }
+}
+}  // namespace
+
 extern "C" {
 bool LoadSizeFile(const char* compressed, size_t size) {
+  writer.reset(Json::StreamWriterBuilder().newStreamWriter());
   caspian::ParseSizeInfo(compressed, size, &info);
+  // Build tree
+  root.id_path = "/";
+  parents[""] = &root;
+  AddSymbolsAndFileNodes(&info);
   return true;
 }
+
+const char* Open(const char* path) {
+  // Returns a string that can be parsed to a JS object.
+  static std::string result;
+  const auto node = parents.find(path);
+
+  if (node != parents.end()) {
+    Json::Value v;
+    OpenIntoJson(node->second, &v, 1);
+    result = ToJson(v);
+    return result.c_str();
+  } else {
+    std::cerr << "Tried to open nonexistent node with path: " << path
+              << std::endl;
+    exit(1);
+  }
 }
+}  // extern "C"

tools/binary_size/libsupersize/caspian/model.cc

@@ -6,8 +6,13 @@
 
 #include "tools/binary_size/libsupersize/caspian/file_format.h"
 
-caspian::Symbol::Symbol() = default;
-caspian::Symbol::Symbol(const Symbol& other) = default;
+using namespace caspian;
 
-caspian::SizeInfo::SizeInfo() = default;
-caspian::SizeInfo::~SizeInfo() = default;
+Symbol::Symbol() = default;
+Symbol::Symbol(const Symbol& other) = default;
+
+TreeNode::TreeNode() = default;
+TreeNode::~TreeNode() = default;
+
+SizeInfo::SizeInfo() = default;
+SizeInfo::~SizeInfo() = default;

tools/binary_size/libsupersize/caspian/model.h

@@ -32,6 +32,28 @@ struct Symbol {
   std::vector<Symbol*>* aliases = nullptr;
 };
 
+struct TreeNode {
+  TreeNode();
+  ~TreeNode();
+
+  std::string_view id_path;
+  const char* src_path = nullptr;
+  const char* component = nullptr;
+  int32_t size = 0;
+  int32_t flags = 0;
+  int32_t short_name_index = 0;
+
+  /*
+    type,
+    numAliases,
+    childStats,
+  */
+
+  std::deque<TreeNode*> children;
+  TreeNode* parent = nullptr;
+  Symbol* symbol = nullptr;
+};
+
 struct SizeInfo {
   SizeInfo();
   ~SizeInfo();

tools/binary_size/libsupersize/static/tree-worker-wasm.js

@@ -127,8 +127,16 @@ function mallocBuffer(buf) {
   return dataHeap;
 }
 
-function freeBuffer(buf) {
-  Module._free(buf.byteOffset);
+var _Open = null;
+function Open(name) {
+  if (_Open === null) {
+    _Open = Module.cwrap('Open', 'number', ['string']);
+  }
+  let stringPtr = _Open(name);
+  // Something has gone wrong if we get back a string longer than 67MB.
+  let ret = JSON.parse(Module.UTF8ToString(stringPtr, 2 ** 26));
+  console.log(ret);
+  return ret;
 }
 
 // Placeholder input name until supplied via setInput()
@@ -144,7 +152,40 @@ async function buildTree(
   let start_time = Date.now();
   console.log(LoadSizeFile(heapBuffer.byteOffset, sizeBuffer.byteLength));
   console.log('Loaded size file in ' + (Date.now() - start_time)/1000.0 + ' seconds');
-  freeBuffer(heapBuffer);
+  Module._free(heapBuffer.byteOffset);
+
+  /**
+   * Creates data to post to the UI thread. Defaults will be used for the root
+   * and percent values if not specified.
+   * @param {{root?:TreeNode,percent?:number,error?:Error}} data Default data
+   * values to post.
+   */
+  function createProgressMessage(data = {}) {
+    let {percent} = data;
+    if (percent == null) {
+      if (meta == null) {
+        percent = 0;
+      } else {
+        percent = Math.max(builder.rootNode.size / meta.total, 0.1);
+      }
+    }
+
+    const message = {
+      root: builder.formatNode(data.root || builder.rootNode),
+      percent,
+      diffMode: meta && meta.diff_mode,
+    };
+    if (data.error) {
+      message.error = data.error.message;
+    }
+    return message;
+  }
+
+  return {
+    root: Open(''),
+    percent: 1.0,
+    diffMode: 0, // diff mode
+  };
 }
 
 /**
@@ -192,9 +233,7 @@ const actions = {
   },
   /** @param {string} path */
   async open(path) {
-    if (!builder) throw new Error('Called open before load');
-    const node = builder.find(path);
-    return builder.formatNode(node);
+    return Open(path);
   },
 };