Fill in the second pass of the parser to analyze hprof files.

The main purpose of the java heap profiler is to obtain a heap dump in
the form of an hprof file and then parse the hprof file. Currently,
there exists a parser that does one pass and keeps track of all
the different instances and fields associated with them such as
object_id, name, etc.

With this CL, we pass over all the instances we just stored. The purpose
if that is that we are able to connect references between two different
instances.


Bug: 1012072
Change-Id: I03180731ab5ce347383f64d1103d5dd750326a96
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1907281
Commit-Queue: ssid <ssid@chromium.org>
Reviewed-by: ssid <ssid@chromium.org>
Cr-Commit-Position: refs/heads/master@{#719942}

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android_unittest.cc

@@ -2,11 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
+
 #include <stddef.h>
 #include <cstdint>
 
-#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
-
 #include "testing/gtest/include/gtest/gtest.h"
 
 namespace tracing {

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_instances_android.cc

@@ -8,40 +8,43 @@
 
 namespace tracing {
 
+Instance::Instance(uint64_t object_id) : object_id(object_id) {}
 Instance::Instance(uint64_t object_id, std::string type_name)
     : type_name(type_name), object_id(object_id) {}
-
 Instance::Instance(uint64_t object_id, uint32_t size)
     : object_id(object_id), size(size) {}
 
 Instance::Instance(uint64_t object_id, uint32_t size, std::string type_name)
     : type_name(type_name), object_id(object_id), size(size) {}
 
+Instance::Instance(const Instance& other) = default;
+
 Instance::~Instance() {}
 
-Instance::Instance(const Instance& other) = default;
+void Instance::AddReference(const std::string& name, uint64_t object_id) {
+  references.push_back({name, object_id});
+}
 
 ClassInstance::ClassInstance(uint64_t object_id,
                              uint64_t class_id,
-                             uint32_t temp_data_position,
-                             uint32_t size)
-    : base_instance(object_id, size),
+                             uint32_t temp_data_position)
+    : base_instance(object_id),
       class_id(class_id),
       temp_data_position(temp_data_position) {}
 
 Field::Field(std::string name, DataType type, uint64_t object_id)
     : name(name), type(type), object_id(object_id) {}
 
-ClassObject::~ClassObject() {}
-
 ClassObject::ClassObject(uint64_t object_id, std::string type_name)
     : base_instance(object_id, type_name) {}
 
+ClassObject::~ClassObject() {}
+
 ObjectArrayInstance::ObjectArrayInstance(uint64_t object_id,
                                          uint64_t class_id,
                                          uint32_t temp_data_position,
                                          uint32_t temp_data_length,
-                                         uint32_t size)
+                                         uint64_t size)
     : base_instance(object_id, size),
       class_id(class_id),
       temp_data_position(temp_data_position),
@@ -50,7 +53,7 @@ ObjectArrayInstance::ObjectArrayInstance(uint64_t object_id,
 PrimitiveArrayInstance::PrimitiveArrayInstance(uint64_t object_id,
                                                DataType type,
                                                std::string type_name,
-                                               uint32_t size)
+                                               uint64_t size)
     : base_instance(object_id, size, type_name), type(type) {}
 
 }  // namespace tracing

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_instances_android.h

@@ -23,13 +23,16 @@ struct COMPONENT_EXPORT(TRACING_CPP) Instance {
     uint64_t referred_from_object_id;
   };
 
+  explicit Instance(uint64_t object_id);
   Instance(uint64_t object_id, std::string type_name);
   Instance(uint64_t object_id, uint32_t size);
   Instance(uint64_t object_id, uint32_t size, std::string type_name);
   Instance(const Instance& other);
   ~Instance();
 
-  // Only set in first pass for ClassObject and PrimitiveArrayInstances.
+  void AddReference(const std::string& name, uint64_t object_id);
+
+  //  Only set in first pass for ClassObject and PrimitiveArrayInstances.
   std::string type_name;
   const uint64_t object_id;  // Always set in first pass.
   uint32_t size;             // Set in first pass except for ClassObject
@@ -41,8 +44,7 @@ struct COMPONENT_EXPORT(TRACING_CPP) Instance {
 struct COMPONENT_EXPORT(TRACING_CPP) ClassInstance {
   ClassInstance(uint64_t object_id,
                 uint64_t class_id,
-                uint32_t temp_data_position,
-                uint32_t size);
+                uint32_t temp_data_position);
 
   Instance base_instance;
   const uint64_t class_id;
@@ -83,7 +85,7 @@ struct COMPONENT_EXPORT(TRACING_CPP) ObjectArrayInstance {
                       uint64_t class_id,
                       uint32_t temp_data_position,
                       uint32_t temp_data_length,
-                      uint32_t size);
+                      uint64_t size);
 
   Instance base_instance;
   const uint64_t class_id;
@@ -104,7 +106,7 @@ struct COMPONENT_EXPORT(TRACING_CPP) PrimitiveArrayInstance {
   PrimitiveArrayInstance(uint64_t object_id,
                          DataType type,
                          std::string type_name,
-                         uint32_t size);
+                         uint64_t size);
   Instance base_instance;
   const DataType type;
 };

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android.cc

@@ -14,6 +14,7 @@
 #include "base/files/file.h"
 #include "base/files/scoped_file.h"
 #include "base/files/scoped_temp_dir.h"
+#include "base/strings/string_number_conversions.h"
 #include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_data_type_android.h"
 #include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_instances_android.h"
 #include "services/tracing/public/cpp/perfetto/perfetto_traced_process.h"
@@ -32,6 +33,11 @@ static DataType GetTypeFromIndex(uint32_t type_index) {
   return static_cast<DataType>(type_index);
 }
 
+static std::string GenerateArrayIndexString(const std::string& type_name,
+                                            uint32_t index) {
+  return type_name + "$" + base::NumberToString(index);
+}
+
 }  // namespace
 
 const std::string& HprofParser::StringReference::GetString() {
@@ -125,12 +131,10 @@ HprofParser::ParseResult HprofParser::ParseClassObjectDumpSubtag() {
       object_id = kInvalidObjectId;
       hprof_->SkipBytesByType(type);
     }
-
     static_fields_size += hprof_->SizeOfType(type_index);
 
     class_obj->static_fields.emplace_back(static_field_name, type, object_id);
   }
-
   class_obj->base_instance.size = static_fields_size;
 
   uint32_t num_instance_fields = hprof_->GetTwoBytes();
@@ -164,8 +168,8 @@ HprofParser::ParseResult HprofParser::ParseClassInstanceDumpSubtag() {
   hprof_->Skip(instance_size);
 
   class_instances_.emplace(
-      object_id, std::make_unique<ClassInstance>(
-                     object_id, class_id, temp_data_position, instance_size));
+      object_id,
+      std::make_unique<ClassInstance>(object_id, class_id, temp_data_position));
 
   return ParseResult::PARSE_SUCCESS;
 }
@@ -208,6 +212,87 @@ HprofParser::ParseResult HprofParser::ParsePrimitiveArrayDumpSubtag() {
   return ParseResult::PARSE_SUCCESS;
 }
 
+HprofParser::ParseResult HprofParser::ResolveClassInstanceReferences() {
+  for (auto& it : class_instances_) {
+    ClassInstance* class_instance = it.second.get();
+
+    auto class_objects_it = class_objects_.find(class_instance->class_id);
+    if (class_objects_it == class_objects_.end()) {
+      parse_stats_.result = ParseResult::OBJECT_ID_NOT_FOUND;
+      return parse_stats_.result;
+    }
+    ClassObject* class_obj = class_objects_it->second.get();
+
+    class_instance->base_instance.size = class_obj->instance_size;
+    class_instance->base_instance.type_name =
+        class_obj->base_instance.type_name;
+
+    hprof_->set_position(class_instance->temp_data_position);
+
+    for (Field f : class_obj->instance_fields) {
+      if (f.type != DataType::OBJECT) {
+        hprof_->SkipBytesByType(f.type);
+        continue;
+      }
+
+      ObjectId id = hprof_->GetId();
+      Instance* base_instance = FindInstance(id);
+
+      // If instance is not found, just move on to the next id without adding
+      // any references.
+      if (!base_instance)
+        continue;
+
+      base_instance->AddReference(f.name,
+                                  class_instance->base_instance.object_id);
+    }
+  }
+
+  return ParseResult::PARSE_SUCCESS;
+}
+
+HprofParser::ParseResult HprofParser::ResolveObjectArrayInstanceReferences() {
+  for (auto& it : object_array_instances_) {
+    ObjectArrayInstance* object_array_instance = it.second.get();
+    hprof_->set_position(object_array_instance->temp_data_position);
+
+    auto class_objects_it =
+        class_objects_.find(object_array_instance->class_id);
+    if (class_objects_it == class_objects_.end()) {
+      parse_stats_.result = ParseResult::OBJECT_ID_NOT_FOUND;
+      return parse_stats_.result;
+    }
+    ClassObject* class_obj = class_objects_it->second.get();
+
+    object_array_instance->base_instance.type_name =
+        class_obj->base_instance.type_name;
+
+    for (uint32_t i = 0; i < object_array_instance->temp_data_length; i++) {
+      ObjectId id = hprof_->GetId();
+      Instance* base_instance = FindInstance(id);
+
+      // If instance is not found, just move on to the next id without adding
+      // any references.
+      if (!base_instance)
+        continue;
+
+      base_instance->AddReference(
+          GenerateArrayIndexString(
+              object_array_instance->base_instance.type_name, i),
+          object_array_instance->base_instance.object_id);
+    }
+  }
+  return ParseResult::PARSE_SUCCESS;
+}
+
+void HprofParser::ModifyClassObjectTypeNames() {
+  for (auto& it : class_objects_) {
+    std::string new_type_name =
+        "java.lang.Class:" + it.second.get()->base_instance.type_name;
+    it.second.get()->base_instance.type_name = new_type_name;
+  }
+}
+
 HprofParser::ParseResult HprofParser::ParseHeapDumpTag(
     uint32_t record_length_) {
   parse_stats_.num_heap_dump_segments++;
@@ -298,6 +383,7 @@ HprofParser::ParseResult HprofParser::ParseHeapDumpTag(
         NOTREACHED();
     }
   }
+
   return ParseResult::PARSE_SUCCESS;
 }
 
@@ -347,6 +433,28 @@ void HprofParser::ParseFileData(const unsigned char* file_data,
         hprof_->Skip(record_length_);
     }
   }
+
+  // Currently we have all instances defined in the hprof file within four
+  // separate id->instance based off instance type (ClassObject, ClassInstance,
+  // ObjectArrayInstance, PrimitiveArrayInstance). The next step is to take
+  // these instances and resolve references between pairs of instances.
+  // We do this specifically for ClassInstances and ObjectArrayInstances.
+  // For ClassInstances, we set a reference between a given class instance
+  // and any instances that are instance variables of the given class instance.
+  // For ObjectArrayInstances, we set a reference between objects within the
+  // object array and the actual object array.
+  if (ResolveClassInstanceReferences() != ParseResult::PARSE_SUCCESS) {
+    return;
+  }
+
+  ParseResult object_array_instance_result =
+      ResolveObjectArrayInstanceReferences();
+  if (object_array_instance_result != ParseResult::PARSE_SUCCESS) {
+    return;
+  }
+
+  ModifyClassObjectTypeNames();
+
   parse_stats_.result = ParseResult::PARSE_SUCCESS;
 }
 
@@ -378,4 +486,29 @@ HprofParser::ParseResult HprofParser::Parse() {
 
   return parse_stats_.result;
 }
+
+Instance* HprofParser::FindInstance(ObjectId id) {
+  auto class_instance_sub_instance_it = class_instances_.find(id);
+  if (class_instance_sub_instance_it != class_instances_.end()) {
+    return &class_instance_sub_instance_it->second->base_instance;
+  }
+
+  auto object_array_sub_instance_it = object_array_instances_.find(id);
+  if (object_array_sub_instance_it != object_array_instances_.end()) {
+    return &object_array_sub_instance_it->second->base_instance;
+  }
+
+  auto primitive_array_sub_instance_it = primitive_array_instances_.find(id);
+  if (primitive_array_sub_instance_it != primitive_array_instances_.end()) {
+    return &primitive_array_sub_instance_it->second->base_instance;
+  }
+
+  auto class_object_sub_instance_it = class_objects_.find(id);
+  if (class_object_sub_instance_it != class_objects_.end()) {
+    return &class_object_sub_instance_it->second->base_instance;
+  }
+
+  return nullptr;
+}
+
 }  // namespace tracing

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android.h

@@ -10,6 +10,7 @@
 
 #include "base/component_export.h"
 #include "base/gtest_prod_util.h"
+#include "base/optional.h"
 #include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
 #include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_instances_android.h"
 
@@ -33,6 +34,7 @@ class COMPONENT_EXPORT(TRACING_CPP) HprofParser {
     PARSE_FAILED,
     FAILED_TO_OPEN_FILE,
     STRING_ID_NOT_FOUND,
+    OBJECT_ID_NOT_FOUND,
   };
 
   struct ParseStats {
@@ -100,7 +102,24 @@ class COMPONENT_EXPORT(TRACING_CPP) HprofParser {
   FRIEND_TEST_ALL_PREFIXES(HprofParserTest, ParseClassInstanceDumpSubtag);
   FRIEND_TEST_ALL_PREFIXES(HprofParserTest, ParseObjectArrayDumpSubtag);
   FRIEND_TEST_ALL_PREFIXES(HprofParserTest, ParsePrimitiveArrayDumpSubtag);
-
+  FRIEND_TEST_ALL_PREFIXES(HprofParserTest, ModifyClassObjectTypeNames);
+
+  FRIEND_TEST_ALL_PREFIXES(HprofParserTest,
+                           BasicResolveClassInstanceReferences);
+  FRIEND_TEST_ALL_PREFIXES(
+      HprofParserTest,
+      MultipleInstanceFieldsResolveClassInstanceReferences);
+  FRIEND_TEST_ALL_PREFIXES(
+      HprofParserTest,
+      MissingObjectReferenceResolveClassInstanceReferences);
+  FRIEND_TEST_ALL_PREFIXES(
+      HprofParserTest,
+      ExistingAndMissingReferencesResolveClassInstanceReferences);
+  FRIEND_TEST_ALL_PREFIXES(HprofParserTest,
+                           BasicResolveObjectArrayInstanceReferences);
+  FRIEND_TEST_ALL_PREFIXES(
+      HprofParserTest,
+      MissingAndExistingReferencesResolveObjectArrayInstanceReferences);
   // Parses hprof data file_data and records metrics in parse_stats_.
   void ParseFileData(const unsigned char* file_data, size_t file_size);
 
@@ -112,6 +131,19 @@ class COMPONENT_EXPORT(TRACING_CPP) HprofParser {
   ParseResult ParsePrimitiveArrayDumpSubtag();
   ParseResult ParseHeapDumpTag(uint32_t record_length_);
 
+  ParseResult ResolveClassInstanceReferences();
+  ParseResult ResolveObjectArrayInstanceReferences();
+
+  // Append java.lang.Class: to ClassObjects to differentiate them from
+  // ClassInstances.
+  void ModifyClassObjectTypeNames();
+
+  // Searches through each of the four instance maps: class_objects_,
+  // class_instances_, object_array_instances_, and primitive_array_instances_
+  // for an instance with given id. If found, return the base instance of the
+  // instance found. If not found, return a null pointer.
+  Instance* FindInstance(ObjectId id);
+
   std::unordered_map<ObjectId, std::unique_ptr<StringReference>> strings_;
 
   std::unordered_map<ObjectId, std::unique_ptr<ClassObject>> class_objects_;