Implement the new track_event json exporter.

This will process perfetto::TracePacket sequences and output the data in
the old legacy json trace format used by chromium.

Change-Id: I8facc1b7371e66c0f444ed6010b379ac407b5c0f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1514834
Commit-Queue: Stephen Nusko <nuskos@chromium.org>
Auto-Submit: Stephen Nusko <nuskos@chromium.org>
Reviewed-by: Eric Seckler <eseckler@chromium.org>
Cr-Commit-Position: refs/heads/master@{#644758}

services/tracing/BUILD.gn

@@ -25,6 +25,8 @@ source_set("lib") {
     "perfetto/perfetto_tracing_coordinator.h",
     "perfetto/producer_host.cc",
     "perfetto/producer_host.h",
+    "perfetto/track_event_json_exporter.cc",
+    "perfetto/track_event_json_exporter.h",
     "recorder.cc",
     "recorder.h",
     "tracing_service.cc",
@@ -112,6 +114,7 @@ source_set("tests") {
       "perfetto/perfetto_tracing_coordinator_unittest.cc",
       "perfetto/test_utils.cc",
       "perfetto/test_utils.h",
+      "perfetto/track_event_json_exporter_unittest.cc",
     ]
 
     deps += [

services/tracing/perfetto/chrome_event_bundle_json_exporter.cc

@@ -13,7 +13,6 @@
 
 namespace tracing {
 namespace {
-using TraceEvent = ::base::trace_event::TraceEvent;
 const char* GetStringFromStringTable(
     const std::unordered_map<int, std::string>& string_table,
     int index) {
@@ -22,132 +21,7 @@ const char* GetStringFromStringTable(
 
   return it->second.c_str();
 }
-void AppendProtoArrayAsJSON(std::string* out,
-                            const perfetto::protos::ChromeTracedValue& array);
 
-void AppendProtoDictAsJSON(std::string* out,
-                           const perfetto::protos::ChromeTracedValue& dict);
-
-void AppendProtoValueAsJSON(std::string* out,
-                            const perfetto::protos::ChromeTracedValue& value) {
-  base::trace_event::TraceEvent::TraceValue json_value;
-  if (value.has_int_value()) {
-    json_value.as_int = value.int_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_INT, json_value, out);
-  } else if (value.has_double_value()) {
-    json_value.as_double = value.double_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_DOUBLE, json_value, out);
-  } else if (value.has_bool_value()) {
-    json_value.as_bool = value.bool_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_BOOL, json_value, out);
-  } else if (value.has_string_value()) {
-    json_value.as_string = value.string_value().c_str();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_STRING, json_value, out);
-  } else if (value.has_nested_type()) {
-    if (value.nested_type() == perfetto::protos::ChromeTracedValue::ARRAY) {
-      AppendProtoArrayAsJSON(out, value);
-      return;
-    } else if (value.nested_type() ==
-               perfetto::protos::ChromeTracedValue::DICT) {
-      AppendProtoDictAsJSON(out, value);
-    } else {
-      NOTREACHED();
-    }
-  } else {
-    NOTREACHED();
-  }
-}
-
-void AppendProtoArrayAsJSON(std::string* out,
-                            const perfetto::protos::ChromeTracedValue& array) {
-  out->append("[");
-
-  bool is_first_entry = true;
-  for (auto& value : array.array_values()) {
-    if (!is_first_entry) {
-      out->append(",");
-    } else {
-      is_first_entry = false;
-    }
-
-    AppendProtoValueAsJSON(out, value);
-  }
-
-  out->append("]");
-}
-
-void AppendProtoDictAsJSON(std::string* out,
-                           const perfetto::protos::ChromeTracedValue& dict) {
-  out->append("{");
-
-  DCHECK_EQ(dict.dict_keys_size(), dict.dict_values_size());
-  for (int i = 0; i < dict.dict_keys_size(); ++i) {
-    if (i != 0) {
-      out->append(",");
-    }
-
-    base::EscapeJSONString(dict.dict_keys(i), true, out);
-    out->append(":");
-
-    AppendProtoValueAsJSON(out, dict.dict_values(i));
-  }
-
-  out->append("}");
-}
-
-void OutputJSONFromArgumentValue(
-    const perfetto::protos::ChromeTraceEvent::Arg& arg,
-    std::string* out) {
-  TraceEvent::TraceValue value;
-  if (arg.has_bool_value()) {
-    value.as_bool = arg.bool_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_BOOL, value, out);
-    return;
-  }
-
-  if (arg.has_uint_value()) {
-    value.as_uint = arg.uint_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_UINT, value, out);
-    return;
-  }
-
-  if (arg.has_int_value()) {
-    value.as_int = arg.int_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_INT, value, out);
-    return;
-  }
-
-  if (arg.has_double_value()) {
-    value.as_double = arg.double_value();
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_DOUBLE, value, out);
-    return;
-  }
-
-  if (arg.has_pointer_value()) {
-    value.as_pointer = reinterpret_cast<void*>(arg.pointer_value());
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_POINTER, value, out);
-    return;
-  }
-
-  if (arg.has_string_value()) {
-    std::string str = arg.string_value();
-    value.as_string = &str[0];
-    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_STRING, value, out);
-    return;
-  }
-
-  if (arg.has_json_value()) {
-    *out += arg.json_value();
-    return;
-  }
-
-  if (arg.has_traced_value()) {
-    AppendProtoDictAsJSON(out, arg.traced_value());
-    return;
-  }
-
-  NOTREACHED();
-}
 }  // namespace
 
 ChromeEventBundleJsonExporter::ChromeEventBundleJsonExporter(
@@ -248,7 +122,7 @@ void ChromeEventBundleJsonExporter::ConstructTraceEventJSONWithBuilder(
 
     auto* maybe_arg = args_builder->MaybeAddArg(arg_name);
     if (maybe_arg) {
-      OutputJSONFromArgumentValue(arg, maybe_arg->mutable_out());
+      OutputJSONFromArgumentProto(arg, maybe_arg->mutable_out());
     }
   }
   // Do not add anything to |trace_event_builder| unless you destroy

services/tracing/perfetto/json_trace_exporter.cc

@@ -19,10 +19,170 @@ namespace tracing {
 
 namespace {
 
+using TraceEvent = ::base::trace_event::TraceEvent;
+
 constexpr size_t kTraceEventBufferSizeInBytes = 100 * 1024;
 
+template <typename Nested>
+void AppendProtoArrayAsJSON(std::string* out, const Nested& array);
+
+template <typename Nested>
+void AppendProtoDictAsJSON(std::string* out, const Nested& dict);
+
+template <typename Nested>
+void AppendProtoValueAsJSON(std::string* out, const Nested& value) {
+  base::trace_event::TraceEvent::TraceValue json_value;
+  if (value.has_int_value()) {
+    json_value.as_int = value.int_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_INT, json_value, out);
+  } else if (value.has_double_value()) {
+    json_value.as_double = value.double_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_DOUBLE, json_value, out);
+  } else if (value.has_bool_value()) {
+    json_value.as_bool = value.bool_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_BOOL, json_value, out);
+  } else if (value.has_string_value()) {
+    json_value.as_string = value.string_value().c_str();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_STRING, json_value, out);
+  } else if (value.has_nested_type()) {
+    if (value.nested_type() == Nested::ARRAY) {
+      AppendProtoArrayAsJSON(out, value);
+      return;
+    } else if (value.nested_type() == Nested::DICT) {
+      AppendProtoDictAsJSON(out, value);
+    } else {
+      NOTREACHED();
+    }
+  } else {
+    NOTREACHED();
+  }
+}
+
+template <typename Nested>
+void AppendProtoArrayAsJSON(std::string* out, const Nested& array) {
+  out->append("[");
+
+  bool is_first_entry = true;
+  for (auto& value : array.array_values()) {
+    if (!is_first_entry) {
+      out->append(",");
+    } else {
+      is_first_entry = false;
+    }
+
+    AppendProtoValueAsJSON(out, value);
+  }
+
+  out->append("]");
+}
+
+template <typename Nested>
+void AppendProtoDictAsJSON(std::string* out, const Nested& dict) {
+  out->append("{");
+
+  DCHECK_EQ(dict.dict_keys_size(), dict.dict_values_size());
+  for (int i = 0; i < dict.dict_keys_size(); ++i) {
+    if (i != 0) {
+      out->append(",");
+    }
+
+    base::EscapeJSONString(dict.dict_keys(i), true, out);
+    out->append(":");
+
+    AppendProtoValueAsJSON(out, dict.dict_values(i));
+  }
+
+  out->append("}");
+}
+
+template <typename Value, typename Nested>
+void OutputJSONFromArgumentValue(const Value& arg,
+                                 const base::Optional<Nested>& nested,
+                                 const base::Optional<std::string>& json_value,
+                                 std::string* out) {
+  TraceEvent::TraceValue value;
+  if (arg.has_bool_value()) {
+    value.as_bool = arg.bool_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_BOOL, value, out);
+    return;
+  }
+
+  if (arg.has_uint_value()) {
+    value.as_uint = arg.uint_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_UINT, value, out);
+    return;
+  }
+
+  if (arg.has_int_value()) {
+    value.as_int = arg.int_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_INT, value, out);
+    return;
+  }
+
+  if (arg.has_double_value()) {
+    value.as_double = arg.double_value();
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_DOUBLE, value, out);
+    return;
+  }
+
+  if (arg.has_pointer_value()) {
+    value.as_pointer = reinterpret_cast<void*>(arg.pointer_value());
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_POINTER, value, out);
+    return;
+  }
+
+  if (arg.has_string_value()) {
+    std::string str = arg.string_value();
+    value.as_string = &str[0];
+    TraceEvent::AppendValueAsJSON(TRACE_VALUE_TYPE_STRING, value, out);
+    return;
+  }
+
+  if (json_value) {
+    *out += *json_value;
+    return;
+  }
+
+  if (nested) {
+    AppendProtoDictAsJSON(out, nested.value());
+    return;
+  }
+
+  NOTREACHED();
+}
 }  // namespace
 
+void OutputJSONFromArgumentProto(
+    const perfetto::protos::ChromeTraceEvent::Arg& arg,
+    std::string* out) {
+  base::Optional<perfetto::protos::ChromeTracedValue> traced_value;
+  if (arg.has_traced_value()) {
+    traced_value = arg.traced_value();
+  }
+  base::Optional<std::string> json_value;
+  if (arg.has_json_value()) {
+    json_value = arg.json_value();
+  }
+  OutputJSONFromArgumentValue<perfetto::protos::ChromeTraceEvent::Arg,
+                              perfetto::protos::ChromeTracedValue>(
+      arg, traced_value, json_value, out);
+}
+
+void OutputJSONFromArgumentProto(const perfetto::protos::DebugAnnotation& arg,
+                                 std::string* out) {
+  base::Optional<perfetto::protos::DebugAnnotation::NestedValue> nested_value;
+  if (arg.has_nested_value()) {
+    nested_value = arg.nested_value();
+  }
+  base::Optional<std::string> json_value;
+  if (arg.has_legacy_json_value()) {
+    json_value = arg.legacy_json_value();
+  }
+  OutputJSONFromArgumentValue<perfetto::protos::DebugAnnotation,
+                              perfetto::protos::DebugAnnotation::NestedValue>(
+      arg, nested_value, json_value, out);
+}
+
 JSONTraceExporter::JSONTraceExporter(
     ArgumentFilterPredicate argument_filter_predicate,
     OnTraceEventJSONCallback callback)
@@ -106,12 +266,12 @@ void JSONTraceExporter::AddChromeLegacyJSONTrace(
   DCHECK(!json_trace.data().empty());
   switch (json_trace.type()) {
     case perfetto::protos::ChromeLegacyJsonTrace::USER_TRACE:
-      *AddJSONTraceEvent() += json_trace.data();
-      return;
-    case perfetto::protos::ChromeLegacyJsonTrace::SYSTEM_TRACE:
       if (!ShouldOutputTraceEvents()) {
         return;
       }
+      *AddJSONTraceEvent() += json_trace.data();
+      return;
+    case perfetto::protos::ChromeLegacyJsonTrace::SYSTEM_TRACE:
       if (legacy_system_trace_events_.empty()) {
         legacy_system_trace_events_ = "{";
       } else {
@@ -371,6 +531,9 @@ JSONTraceExporter::ScopedJSONTraceEventAppender::ScopedJSONTraceEventAppender(
     JSONTraceExporter::ScopedJSONTraceEventAppender&& move) {
   out_ = move.out_;
   phase_ = move.phase_;
+  added_args_ = move.added_args_;
+  event_name_ = std::move(move.event_name_);
+  category_group_name_ = std::move(move.category_group_name_);
   argument_filter_predicate_ = std::move(move.argument_filter_predicate_);
   // We null out the string so that the destructor knows not to append the
   // closing brace for the json.

services/tracing/perfetto/json_trace_exporter.h

@@ -21,12 +21,20 @@ namespace perfetto {
 namespace protos {
 class ChromeLegacyJsonTrace;
 class ChromeMetadata;
+class ChromeTraceEvent_Arg;
+class DebugAnnotation;
 class TraceStats;
 }  // namespace protos
 }  // namespace perfetto
 
 namespace tracing {
 
+void OutputJSONFromArgumentProto(
+    const perfetto::protos::ChromeTraceEvent_Arg& arg,
+    std::string* out);
+void OutputJSONFromArgumentProto(const perfetto::protos::DebugAnnotation& arg,
+                                 std::string* out);
+
 // Converts proto-encoded trace data into the legacy JSON trace format.
 // Conversion happens on-the-fly as new trace packets are received.
 class JSONTraceExporter {

services/tracing/perfetto/track_event_json_exporter.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef SERVICES_TRACING_PERFETTO_TRACK_EVENT_JSON_EXPORTER_H_
+#define SERVICES_TRACING_PERFETTO_TRACK_EVENT_JSON_EXPORTER_H_
+
+#include <string>
+#include <unordered_map>
+
+#include "services/tracing/perfetto/json_trace_exporter.h"
+
+namespace perfetto {
+namespace protos {
+class ChromeTracePacket;
+class DebugAnnotation;
+class TaskExecution;
+class TrackEvent;
+class TrackEvent_LegacyEvent;
+}  // namespace protos
+}  // namespace perfetto
+
+namespace tracing {
+
+class TrackEventJSONExporter : public JSONTraceExporter {
+ public:
+  TrackEventJSONExporter(ArgumentFilterPredicate argument_filter_predicate,
+                         OnTraceEventJSONCallback callback);
+
+  ~TrackEventJSONExporter() override;
+
+ protected:
+  void ProcessPackets(
+      const std::vector<perfetto::TracePacket>& packets) override;
+
+ private:
+  struct ProducerWriterState {
+    ProducerWriterState(uint32_t sequence_id);
+    ProducerWriterState(uint32_t sequence_id,
+                        bool emitted_process,
+                        bool emitted_thread,
+                        bool incomplete);
+    ~ProducerWriterState();
+
+    // 0 is an invalid sequence_id.
+    uint32_t trusted_packet_sequence_id = 0;
+
+    int32_t pid = -1;
+    int32_t tid = -1;
+    int64_t time_us = -1;
+    int64_t thread_time_us = -1;
+
+    // We only want to add metadata events about the process or threads once.
+    // This is to prevent duplicate events in the json since the packets
+    // containing this data are periodically emitted and so would occur
+    // frequently if not suppressed.
+    bool emitted_process_metadata = false;
+    bool emitted_thread_metadata = false;
+
+    // Until we see a TracePacket that will initialize our state we will skip
+    // all data besides stateful information. Once we've been reset on the same
+    // sequence or started a new sequence this will become false and we will
+    // start emitting events again.
+    bool incomplete = true;
+
+    std::unordered_map<uint32_t, std::string> interned_event_categories_;
+    std::unordered_map<uint32_t, std::pair<std::string, std::string>>
+        interned_source_locations_;
+    std::unordered_map<uint32_t, std::string> interned_legacy_event_names_;
+    std::unordered_map<uint32_t, std::string> interned_debug_annotation_names_;
+  };
+
+  // Packet sequences are given in order so when we encounter a new one we need
+  // to reset all the interned state and per sequence info.
+  void StartNewState(uint32_t trusted_packet_sequence_id, bool state_cleared);
+  // When we encounter a request to reset our incremental state this will clear
+  // out the |current_state_| leaving only the required persistent data (like
+  // |emitted_process_metadata|) the same.
+  void ResetIncrementalState();
+
+  // Given our |current_state_| and an |event| we determine the timestamp (or
+  // thread timestamp) we should output to the json.
+  int64_t ComputeTimeUs(const perfetto::protos::TrackEvent& event);
+  base::Optional<int64_t> ComputeThreadTimeUs(
+      const perfetto::protos::TrackEvent& event);
+
+  // Gather all the interned strings of different types.
+  void HandleInternedData(const perfetto::protos::ChromeTracePacket& packet);
+
+  // New typed messages that are part of the oneof in TracePacket.
+  void HandleProcessDescriptor(
+      const perfetto::protos::ChromeTracePacket& packet);
+  void HandleThreadDescriptor(
+      const perfetto::protos::ChromeTracePacket& packet);
+  void HandleChromeEvents(const perfetto::protos::ChromeTracePacket& packet);
+  void HandleTrackEvent(const perfetto::protos::ChromeTracePacket& packet);
+
+  // New typed args handlers go here. Used inside HandleTrackEvent to process
+  // args.
+  void HandleDebugAnnotation(
+      const perfetto::protos::DebugAnnotation& debug_annotation,
+      ArgumentBuilder* args_builder);
+  void HandleTaskExecution(const perfetto::protos::TaskExecution& task,
+                           ArgumentBuilder* args_builder);
+
+  // Used to handle the LegacyEvent message found inside the TrackEvent proto.
+  base::Optional<ScopedJSONTraceEventAppender> HandleLegacyEvent(
+      const perfetto::protos::TrackEvent_LegacyEvent& event,
+      const std::string& categories,
+      int64_t timestamp_us);
+
+  // Tracks all the interned state in the current sequence.
+  ProducerWriterState current_state_;
+};
+}  // namespace tracing
+#endif  // SERVICES_TRACING_PERFETTO_TRACK_EVENT_JSON_EXPORTER_H_