Create empty parser for hprof files to be called by the java heap profiler

The main purpose of the java heap profiler is to obtain a heap dump in
the form of an hprof file and then parse the hprof file.

Currently,there exists a way to obtain the heap dump but there is no
way to parse the hprof file. This CL includes a barebones parser for
these hprof files.

Bug: 1012072
Change-Id: I7a5d19ba8a4c0284fcae41d7057ad44a0b0acdab
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1861103
Commit-Queue: George Zhang <zhanggeorge@google.com>
Reviewed-by: Tommy Nyquist <nyquist@chromium.org>
Reviewed-by: ssid <ssid@chromium.org>
Cr-Commit-Position: refs/heads/master@{#712717}

services/tracing/BUILD.gn

@@ -132,6 +132,8 @@ source_set("tests") {
     "perfetto/json_trace_exporter_unittest.cc",
     "perfetto/perfetto_integration_unittest.cc",
     "perfetto/track_event_json_exporter_unittest.cc",
+    "public/cpp/perfetto/java_heap_profiler/hprof_buffer_android_unittest.cc",
+    "public/cpp/perfetto/java_heap_profiler/hprof_parser_android_unittest.cc",
     "public/cpp/perfetto/task_runner_unittest.cc",
     "public/cpp/perfetto/trace_event_data_source_unittest.cc",
     "public/cpp/perfetto/traced_value_proto_writer_unittest.cc",

services/tracing/public/cpp/BUILD.gn

@@ -34,6 +34,10 @@ if (!is_nacl && !is_ios) {
       "perfetto/dummy_producer.cc",
       "perfetto/dummy_producer.h",
       "perfetto/interning_index.h",
+      "perfetto/java_heap_profiler/hprof_buffer_android.cc",
+      "perfetto/java_heap_profiler/hprof_buffer_android.h",
+      "perfetto/java_heap_profiler/hprof_parser_android.cc",
+      "perfetto/java_heap_profiler/hprof_parser_android.h",
       "perfetto/java_heap_profiler/java_heap_profiler_android.cc",
       "perfetto/java_heap_profiler/java_heap_profiler_android.h",
       "perfetto/perfetto_config.cc",

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
+
+#include "base/logging.h"
+
+namespace tracing {
+
+HprofBuffer::HprofBuffer(const unsigned char* data, size_t size)
+    : data_(data), size_(size) {}
+
+uint32_t HprofBuffer::GetOneByte() {
+  return GetUInt32FromBytes(1);
+}
+
+uint32_t HprofBuffer::GetTwoBytes() {
+  return GetUInt32FromBytes(2);
+}
+
+uint32_t HprofBuffer::GetFourBytes() {
+  return GetUInt32FromBytes(4);
+}
+
+uint64_t HprofBuffer::GetId() {
+  return GetUInt64FromBytes(object_id_size_in_bytes_);
+}
+
+bool HprofBuffer::HasRemaining() {
+  return data_position_ < size_;
+}
+
+void HprofBuffer::set_id_size(unsigned id_size) {
+  DCHECK(id_size == 4 || id_size == 8);
+  object_id_size_in_bytes_ = id_size;
+}
+
+void HprofBuffer::set_position(size_t new_position) {
+  DCHECK(new_position <= size_ && new_position >= 0);
+  data_position_ = new_position;
+}
+
+// Skips |delta| bytes in the buffer.
+void HprofBuffer::Skip(uint32_t delta) {
+  set_position(data_position_ + delta);
+}
+
+unsigned char HprofBuffer::GetByte() {
+  DCHECK(HasRemaining());
+  unsigned char byte = data_[data_position_];
+  ++data_position_;
+  return byte;
+}
+
+// Read in the next |num_bytes| as an uint32_t.
+uint32_t HprofBuffer::GetUInt32FromBytes(size_t num_bytes) {
+  uint32_t val = 0;
+  for (size_t i = 0; i < num_bytes; ++i) {
+    val = (val << 8) + GetByte();
+  }
+  return val;
+}
+
+// Read in the next |num_bytes| as an uint64_t.
+uint64_t HprofBuffer::GetUInt64FromBytes(size_t num_bytes) {
+  uint64_t val = 0;
+  for (size_t i = 0; i < num_bytes; ++i) {
+    val = (val << 8) + GetByte();
+  }
+  return val;
+}
+}  // namespace tracing

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SERVICES_TRACING_PUBLIC_CPP_PERFETTO_JAVA_HEAP_PROFILER_HPROF_BUFFER_ANDROID_H_
+#define SERVICES_TRACING_PUBLIC_CPP_PERFETTO_JAVA_HEAP_PROFILER_HPROF_BUFFER_ANDROID_H_
+
+#include <stddef.h>
+#include <cstdint>
+
+#include "base/component_export.h"
+#include "base/macros.h"
+
+namespace tracing {
+
+// Helper class that has methods to help parse the hprof file data passed in.
+// Works by accessing byte one at a time through data_[data_position_] while
+// also incrementing |data_position_| after reading a byte.
+class COMPONENT_EXPORT(TRACING_CPP) HprofBuffer {
+ public:
+  HprofBuffer(const unsigned char* data, size_t size);
+  HprofBuffer(const HprofBuffer&) = delete;
+  HprofBuffer& operator=(const HprofBuffer&) = delete;
+
+  uint32_t GetOneByte();
+  uint32_t GetTwoBytes();
+  uint32_t GetFourBytes();
+  uint64_t GetId();
+  bool HasRemaining();
+  void set_id_size(unsigned id_size);
+  void set_position(size_t new_position);
+
+  // Skips |delta| bytes in the buffer.
+  void Skip(uint32_t delta);
+
+ private:
+  unsigned char GetByte();
+
+  // Read in the next |num_bytes| as an uint32_t.
+  uint32_t GetUInt32FromBytes(size_t num_bytes);
+
+  // Read in the next |num_bytes| as an uint64_t.
+  uint64_t GetUInt64FromBytes(size_t num_bytes);
+
+  const unsigned char* const data_;
+  const size_t size_;
+  size_t data_position_ = 0;
+  // The ID size in bytes of the objects in the hprof, valid values are 4 and 8.
+  unsigned object_id_size_in_bytes_ = 4;
+};
+
+}  // namespace tracing
+
+#endif  // SERVICES_TRACING_PUBLIC_CPP_PERFETTO_JAVA_HEAP_PROFILER_HPROF_BUFFER_ANDROID_H_

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stddef.h>
+#include <cstdint>
+
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace tracing {
+
+TEST(HprofBufferTest, VerifyBasicGetBytes) {
+  unsigned char file_data[7]{1, 1, 1, 1, 1, 1, 1};
+  HprofBuffer hprof(file_data, 7);
+  EXPECT_EQ(hprof.GetOneByte(), (uint32_t)1);
+  EXPECT_EQ(hprof.GetTwoBytes(), (uint32_t)257);
+  EXPECT_EQ(hprof.GetFourBytes(), (uint32_t)16843009);
+  EXPECT_EQ(hprof.HasRemaining(), false);
+}
+
+TEST(HprofBufferTest, VerifyBasicGetId) {
+  unsigned char file_data[12]{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
+  HprofBuffer hprof(file_data, 12);
+  EXPECT_EQ(hprof.GetId(), (uint64_t)16843009);
+  hprof.set_id_size(8);
+  EXPECT_EQ(hprof.GetId(), (uint64_t)72340172838076673);
+  EXPECT_EQ(hprof.HasRemaining(), false);
+}
+
+TEST(HprofBufferTest, VerifyBasicPositionalMethods) {
+  unsigned char file_data[4]{1, 2, 3, 4};
+  HprofBuffer hprof(file_data, 4);
+  EXPECT_EQ(hprof.GetOneByte(), (uint32_t)1);
+  hprof.Skip(2);
+  EXPECT_EQ(hprof.GetOneByte(), (uint32_t)4);
+  EXPECT_EQ(hprof.HasRemaining(), false);
+
+  hprof.set_position(1);
+  EXPECT_EQ(hprof.GetOneByte(), (uint32_t)2);
+}
+}  // namespace tracing

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "hprof_parser_android.h"
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fstream>
+
+#include "base/android/java_heap_dump_generator.h"
+#include "base/files/file.h"
+#include "base/files/scoped_file.h"
+#include "base/files/scoped_temp_dir.h"
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
+#include "services/tracing/public/cpp/perfetto/perfetto_traced_process.h"
+
+namespace tracing {
+
+HprofParser::HprofParser(const std::string& fp) : file_path_(fp) {}
+
+HprofParser::ParseStats::ParseStats() {}
+
+void HprofParser::ParseFileData(const unsigned char* file_data,
+                                size_t file_size) {
+  HprofBuffer hprof(file_data, file_size);
+
+  uint32_t id_size;
+  // Skip all leading 0s until we find the |id_size|.
+  while (hprof.GetOneByte() != 0 && hprof.HasRemaining()) {
+  }
+  id_size = hprof.GetFourBytes();
+  hprof.set_id_size(id_size);
+
+  hprof.Skip(4);  // hightime
+  hprof.Skip(4);  // lowtime
+  while (hprof.HasRemaining()) {
+    uint32_t tag = hprof.GetOneByte();
+    hprof.Skip(4);  // time
+    uint32_t record_length = hprof.GetFourBytes();
+
+    switch (tag) {
+      default:
+        // Ignore any other tags that we either don't know about or don't
+        // care about. For now, Skip everything.
+        // TODO(zhanggeorge): Add specific parsing per tag.
+        hprof.Skip(record_length);
+        break;
+    }
+  }
+  parse_stats_.result = HprofParser::ParseResult::PARSE_SUCCESS;
+}
+
+HprofParser::ParseResult HprofParser::Parse() {
+  base::ScopedFD fd(open(file_path_.c_str(), O_RDONLY));
+  if (!fd.is_valid()) {
+    parse_stats_.result = HprofParser::ParseResult::FAILED_TO_OPEN_FILE;
+    return parse_stats_.result;
+  }
+
+  struct stat file_stats;
+  if (stat(file_path_.c_str(), &file_stats) < 0) {
+    parse_stats_.result = HprofParser::ParseResult::FAILED_TO_OPEN_FILE;
+    return parse_stats_.result;
+  }
+
+  void* file_data =
+      mmap(0, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd.get(), 0);
+  if (file_data == MAP_FAILED) {
+    parse_stats_.result = HprofParser::ParseResult::FAILED_TO_OPEN_FILE;
+    return parse_stats_.result;
+  }
+
+  ParseFileData(reinterpret_cast<const unsigned char*>(file_data),
+                file_stats.st_size);
+
+  int res = munmap(file_data, file_stats.st_size);
+  DCHECK_EQ(res, 0);
+
+  return parse_stats_.result;
+}
+}  // namespace tracing

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SERVICES_TRACING_PUBLIC_CPP_PERFETTO_JAVA_HEAP_PROFILER_HPROF_PARSER_ANDROID_H_
+#define SERVICES_TRACING_PUBLIC_CPP_PERFETTO_JAVA_HEAP_PROFILER_HPROF_PARSER_ANDROID_H_
+
+#include <string>
+
+#include "base/component_export.h"
+
+namespace tracing {
+
+// This class takes in a temporary file_path where Java API endpoint
+// Debug.dumpHprofData() dumps hprof data to. This file is then parsed for
+// references between different instances. The format is defined by said method.
+// Refer to:
+// https://docs.google.com/document/d/1frGMt8Ro7C6fjbDscImdxHVsFAbSH1mjdETWisZjzVE
+// for more information on the file format and
+// https://developer.android.com/reference/android/os/Debug#dumpHprofData(java.lang.String)
+// for more information on the endpoint.
+class COMPONENT_EXPORT(TRACING_CPP) HprofParser {
+ public:
+  enum ParseResult {
+    PARSE_SUCCESS,
+    PARSE_FAILED,
+    FAILED_TO_OPEN_FILE,
+  };
+
+  struct ParseStats {
+    ParseStats();
+    ParseStats(const ParseStats&) = delete;
+    ParseStats& operator=(const ParseStats&) = delete;
+
+    // Returns the result of the parser
+    ParseResult result = PARSE_FAILED;
+  };
+
+  HprofParser(const std::string& file_path);
+  HprofParser(const HprofParser&) = delete;
+  HprofParser& operator=(const HprofParser&) = delete;
+
+  // This method should only be called after Parse() has been called.
+  const ParseStats& parse_stats() { return parse_stats_; }
+
+  // First opens the file at |file_path_| then passes the data to ParseFileData
+  // to parse and record metrics. The hprof file is generated by
+  // Debug.dumpHprofData().
+  // This method should only ever be run once.
+  ParseResult Parse();
+
+ private:
+  // Parses hprof data file_data and records metrics in parse_stats_.
+  void ParseFileData(const unsigned char* file_data, size_t file_size);
+
+  const std::string file_path_;
+  ParseStats parse_stats_;
+};
+
+}  // namespace tracing
+
+#endif  // SERVICES_TRACING_PUBLIC_CPP_PERFETTO_JAVA_HEAP_PROFILER_HPROF_PARSER_ANDROID_H_

services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android.h"
+#include "base/android/java_heap_dump_generator.h"
+#include "base/files/file_util.h"
+#include "base/files/scoped_temp_dir.h"
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_buffer_android.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace tracing {
+
+TEST(HprofParserTest, ValidateEmptyParser) {
+  base::ScopedTempDir temp_dir;
+  if (!temp_dir.CreateUniqueTempDir()) {
+    VLOG(0) << "Failed to create unique temporary directory.";
+    return;
+  }
+  const std::string file_path_str =
+      temp_dir.GetPath().Append("temp_hprof.hprof").value();
+
+  base::android::WriteJavaHeapDumpToPath(file_path_str);
+  HprofParser parser(file_path_str);
+  parser.Parse();
+  EXPECT_EQ(parser.parse_stats().result,
+            HprofParser::ParseResult::PARSE_SUCCESS);
+}
+
+TEST(HprofParserTest, InvalidPathWithNoDump) {
+  HprofParser parser("invalid_file");
+  parser.Parse();
+  EXPECT_EQ(parser.parse_stats().result,
+            HprofParser::ParseResult::FAILED_TO_OPEN_FILE);
+}
+
+}  // namespace tracing

services/tracing/public/cpp/perfetto/java_heap_profiler/java_heap_profiler_android.cc

@@ -6,6 +6,7 @@
 
 #include "base/android/java_heap_dump_generator.h"
 #include "base/files/scoped_temp_dir.h"
+#include "services/tracing/public/cpp/perfetto/java_heap_profiler/hprof_parser_android.h"
 #include "services/tracing/public/cpp/perfetto/perfetto_traced_process.h"
 
 namespace tracing {
@@ -32,6 +33,10 @@ void JavaHeapProfiler::StartTracing(
   base::android::WriteJavaHeapDumpToPath(file_path);
 
   // TODO(zhanggeorge): Convert heap dump and write to trace.
+  HprofParser parser(file_path);
+  parser.Parse();
+  DCHECK_EQ(parser.parse_stats().result,
+            HprofParser::ParseResult::PARSE_SUCCESS);
 }
 
 void JavaHeapProfiler::StopTracing(base::OnceClosure stop_complete_callback) {