perfetto: Use TraceProcessor for proto2json conversion

TraceProcessor replaces the existing (unmaintainable) proto2json
converter and adds features like combining of BEGIN/END events
and support for heap profiles.

Also updates a devtools test that was expecting separate begin/end
events where TraceProcessor now combines them into a complete event.

We have reduced the trace procesor's binary size overhead
by selectively excluding features Chrome doesn't need from >400kB to
~50kB. We could probably save a couple more kB if we invest even more
eng time and accept the maintenance burden for this in trace processor,
however, we feel that we've reached an acceptable state that balances
binary size & code health.

Binary-Size: see paragraph above.
Bug: 1006770
Change-Id: Ia7f6b3a784c1a9c40574e1dcd80eca1ca2dfb03f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1746258
Commit-Queue: Eric Seckler <eseckler@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Reviewed-by: oysteine <oysteine@chromium.org>
Reviewed-by: Sami Kyöstilä <skyostil@chromium.org>
Cr-Commit-Position: refs/heads/master@{#713593}

services/tracing/BUILD.gn

@@ -16,14 +16,10 @@ source_set("lib") {
   sources = [
     "perfetto/consumer_host.cc",
     "perfetto/consumer_host.h",
-    "perfetto/json_trace_exporter.cc",
-    "perfetto/json_trace_exporter.h",
     "perfetto/perfetto_service.cc",
     "perfetto/perfetto_service.h",
     "perfetto/producer_host.cc",
     "perfetto/producer_host.h",
-    "perfetto/track_event_json_exporter.cc",
-    "perfetto/track_event_json_exporter.h",
     "tracing_service.cc",
     "tracing_service.h",
   ]
@@ -40,31 +36,40 @@ source_set("lib") {
   ]
 
   deps = [
-    "//third_party/perfetto/protos/perfetto/trace/chrome:minimal_complete_lite",
+    "//third_party/perfetto:libproto_to_json",
   ]
 }
 
-executable("trace_json_exporter") {
+source_set("json_trace_exporter") {
   sources = [
-    "perfetto/json_exporter_main.cc",
     "perfetto/json_trace_exporter.cc",
     "perfetto/json_trace_exporter.h",
     "perfetto/track_event_json_exporter.cc",
     "perfetto/track_event_json_exporter.h",
   ]
-
-  configs += [ "//build/config/compiler:rtti" ]
-
   deps = [
     "//base",
     "//third_party/perfetto:libperfetto",
-    "//third_party/perfetto/include/perfetto/protozero:protozero",
     "//third_party/perfetto/protos/perfetto/common:lite",
     "//third_party/perfetto/protos/perfetto/trace:lite",
     "//third_party/perfetto/protos/perfetto/trace/chrome:lite",
     "//third_party/perfetto/protos/perfetto/trace/chrome:minimal_complete_lite",
     "//third_party/perfetto/protos/perfetto/trace/interned_data:lite",
     "//third_party/perfetto/protos/perfetto/trace/track_event:lite",
+  ]
+}
+
+executable("trace_json_exporter") {
+  sources = [
+    "perfetto/json_exporter_main.cc",
+  ]
+
+  configs += [ "//build/config/compiler:rtti" ]
+
+  deps = [
+    ":json_trace_exporter",
+    "//base",
+    "//third_party/perfetto:libperfetto",
     "//third_party/perfetto/src/protozero:protozero",
   ]
 }
@@ -145,6 +150,7 @@ source_set("tests") {
   }
 
   deps = [
+    ":json_trace_exporter",
     ":lib",
     ":test_utils",
     "//base",

services/tracing/perfetto/consumer_host.cc

@@ -24,13 +24,15 @@
 #include "mojo/public/cpp/bindings/self_owned_receiver.h"
 #include "mojo/public/cpp/bindings/strong_binding.h"
 #include "mojo/public/cpp/system/wait.h"
-#include "services/tracing/perfetto/json_trace_exporter.h"
 #include "services/tracing/perfetto/perfetto_service.h"
-#include "services/tracing/perfetto/track_event_json_exporter.h"
 #include "services/tracing/public/cpp/trace_event_args_whitelist.h"
+#include "third_party/perfetto/include/perfetto/ext/trace_processor/export_json.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/observable_events.h"
+#include "third_party/perfetto/include/perfetto/ext/tracing/core/slice.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/trace_packet.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/trace_stats.h"
+#include "third_party/perfetto/include/perfetto/trace_processor/basic_types.h"
+#include "third_party/perfetto/include/perfetto/trace_processor/trace_processor_storage.h"
 #include "third_party/perfetto/include/perfetto/tracing/core/trace_config.h"
 #include "third_party/perfetto/protos/perfetto/config/trace_config.pb.h"
 
@@ -40,6 +42,44 @@ namespace {
 
 const int32_t kEnableTracingTimeoutSeconds = 10;
 
+class JsonStringOutputWriter
+    : public perfetto::trace_processor::json::OutputWriter {
+ public:
+  using FlushCallback =
+      base::RepeatingCallback<void(std::string json, bool has_more)>;
+
+  JsonStringOutputWriter(FlushCallback flush_callback)
+      : flush_callback_(std::move(flush_callback)) {
+    buffer_.reserve(kBufferReserveCapacity);
+  }
+
+  ~JsonStringOutputWriter() override {
+    flush_callback_.Run(std::move(buffer_), false);
+  }
+
+  perfetto::trace_processor::util::Status AppendString(
+      const std::string& string) override {
+    buffer_ += string;
+    if (buffer_.size() > kBufferLimitInBytes) {
+      flush_callback_.Run(std::move(buffer_), true);
+      // Reset the buffer_ after moving it above.
+      buffer_.clear();
+      buffer_.reserve(kBufferReserveCapacity);
+    }
+    return perfetto::trace_processor::util::OkStatus();
+  }
+
+ private:
+  static constexpr size_t kBufferLimitInBytes = 100 * 1024;
+  // Since we write each string before checking the limit, we'll always go
+  // slightly over and hence we reserve some extra space to avoid most
+  // reallocs.
+  static constexpr size_t kBufferReserveCapacity = kBufferLimitInBytes * 5 / 4;
+
+  FlushCallback flush_callback_;
+  std::string buffer_;
+};
+
 }  // namespace
 
 class ConsumerHost::StreamWriter {
@@ -291,7 +331,7 @@ void ConsumerHost::TracingSession::OnTracingDisabled() {
 
   tracing_session_client_->OnTracingDisabled();
 
-  if (json_trace_exporter_) {
+  if (trace_processor_) {
     host_->consumer_endpoint()->ReadBuffers();
   }
 
@@ -343,6 +383,24 @@ void ConsumerHost::TracingSession::DisableTracingAndEmitJson(
                      weak_factory_.GetWeakPtr()),
       base::SequencedTaskRunnerHandle::Get());
 
+  if (privacy_filtering_enabled) {
+    // For filtering/whitelisting to be possible at JSON export time,
+    // filtering must not have been enabled during proto emission time
+    // (or there's nothing to pass through the whitelist).
+    DCHECK(!privacy_filtering_enabled_);
+    privacy_filtering_enabled_ = true;
+  }
+
+  json_agent_label_filter_ = agent_label_filter;
+
+  trace_processor_ =
+      perfetto::trace_processor::TraceProcessorStorage::CreateInstance(
+          perfetto::trace_processor::Config());
+
+  DisableTracing();
+}
+
+void ConsumerHost::TracingSession::ExportJson() {
   // In legacy backend, the trace event agent sets the predicate used by
   // TraceLog. For perfetto backend, ensure that predicate is always set
   // before creating the exporter. The agent can be created later than this
@@ -356,34 +414,55 @@ void ConsumerHost::TracingSession::DisableTracingAndEmitJson(
         base::BindRepeating(&IsMetadataWhitelisted));
   }
 
-  JSONTraceExporter::ArgumentFilterPredicate arg_filter_predicate;
-  JSONTraceExporter::MetadataFilterPredicate metadata_filter_predicate;
-  if (privacy_filtering_enabled) {
-    // For filtering/whitelisting to be possible at JSON export time,
-    // filtering must not have been enabled during proto emission time
-    // (or there's nothing to pass through the whitelist).
-    DCHECK(!privacy_filtering_enabled_);
+  perfetto::trace_processor::json::ArgumentFilterPredicate argument_filter;
+  perfetto::trace_processor::json::MetadataFilterPredicate metadata_filter;
+  perfetto::trace_processor::json::LabelFilterPredicate label_filter;
+
+  if (privacy_filtering_enabled_) {
     auto* trace_log = base::trace_event::TraceLog::GetInstance();
-    arg_filter_predicate = trace_log->GetArgumentFilterPredicate();
-    metadata_filter_predicate = trace_log->GetMetadataFilterPredicate();
+    base::trace_event::ArgumentFilterPredicate argument_filter_predicate =
+        trace_log->GetArgumentFilterPredicate();
+    argument_filter =
+        [argument_filter_predicate](
+            const char* category_group_name, const char* event_name,
+            perfetto::trace_processor::json::ArgumentNameFilterPredicate*
+                name_filter) {
+          base::trace_event::ArgumentNameFilterPredicate name_filter_predicate;
+          bool result = argument_filter_predicate.Run(
+              category_group_name, event_name, &name_filter_predicate);
+          if (name_filter_predicate) {
+            *name_filter = [name_filter_predicate](const char* arg_name) {
+              return name_filter_predicate.Run(arg_name);
+            };
+          }
+          return result;
+        };
+    base::trace_event::MetadataFilterPredicate metadata_filter_predicate =
+        trace_log->GetMetadataFilterPredicate();
+    metadata_filter = [metadata_filter_predicate](const char* metadata_name) {
+      return metadata_filter_predicate.Run(metadata_name);
+    };
   }
-  json_trace_exporter_ = std::make_unique<TrackEventJSONExporter>(
-      std::move(arg_filter_predicate), std::move(metadata_filter_predicate),
-      base::BindRepeating(&ConsumerHost::TracingSession::OnJSONTraceData,
-                          base::Unretained(this)));
 
-  json_trace_exporter_->set_label_filter(agent_label_filter);
+  if (!json_agent_label_filter_.empty()) {
+    label_filter = [this](const char* label) {
+      return strcmp(label, json_agent_label_filter_.c_str()) == 0;
+    };
+  }
 
-  DisableTracing();
+  JsonStringOutputWriter output_writer(base::BindRepeating(
+      &ConsumerHost::TracingSession::OnJSONTraceData, base::Unretained(this)));
+  auto status = perfetto::trace_processor::json::ExportJson(
+      trace_processor_.get(), &output_writer, argument_filter, metadata_filter,
+      label_filter);
+  DCHECK(status.ok()) << status.message();
 }
 
-void ConsumerHost::TracingSession::OnJSONTraceData(
-    std::string* json,
-    base::DictionaryValue* metadata,
-    bool has_more) {
+void ConsumerHost::TracingSession::OnJSONTraceData(std::string json,
+                                                   bool has_more) {
   auto slices = std::make_unique<StreamWriter::Slices>();
   slices->push_back(std::string());
-  slices->back().swap(*json);
+  slices->back().swap(json);
   read_buffers_stream_writer_.Post(FROM_HERE, &StreamWriter::WriteToStream,
                                    std::move(slices), has_more);
 
@@ -396,10 +475,40 @@ void ConsumerHost::TracingSession::OnTraceData(
     std::vector<perfetto::TracePacket> packets,
     bool has_more) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
-  if (json_trace_exporter_) {
-    json_trace_exporter_->OnTraceData(std::move(packets), has_more);
+  if (trace_processor_) {
+    // Calculate space needed for trace chunk. Each packet has a preamble and
+    // payload size.
+    size_t max_size = packets.size() * perfetto::TracePacket::kMaxPreambleBytes;
+    for (const auto& packet : packets) {
+      max_size += packet.size();
+    }
+
+    // Copy packets into a trace file chunk.
+    size_t position = 0;
+    std::unique_ptr<uint8_t[]> data(new uint8_t[max_size]);
+    for (perfetto::TracePacket& packet : packets) {
+      char* preamble;
+      size_t preamble_size;
+      std::tie(preamble, preamble_size) = packet.GetProtoPreamble();
+      DCHECK_LT(position + preamble_size, max_size);
+      memcpy(&data[position], preamble, preamble_size);
+      position += preamble_size;
+      for (const perfetto::Slice& slice : packet.slices()) {
+        DCHECK_LT(position + slice.size, max_size);
+        memcpy(&data[position], slice.start, slice.size);
+        position += slice.size;
+      }
+    }
+
+    auto status = trace_processor_->Parse(std::move(data), position);
+    // TODO(eseckler): There's no way to propagate this error at the moment - If
+    // one occurs on production builds, we silently ignore it and will end up
+    // producing an empty JSON result.
+    DCHECK(status.ok()) << status.message();
     if (!has_more) {
-      json_trace_exporter_.reset();
+      trace_processor_->NotifyEndOfFile();
+      ExportJson();
+      trace_processor_.reset();
     }
     return;
   }

services/tracing/perfetto/consumer_host.h

@@ -23,17 +23,18 @@
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/consumer.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/tracing_service.h"
 
-namespace base {
-class DictionaryValue;
-}
-
 namespace service_manager {
 struct BindSourceInfo;
 }  // namespace service_manager
 
+namespace perfetto {
+namespace trace_processor {
+class TraceProcessorStorage;
+}  // namespace trace_processor
+}  // namespace perfetto
+
 namespace tracing {
 
-class JSONTraceExporter;
 class PerfettoService;
 
 // This is a Mojo interface which enables any client
@@ -89,9 +90,8 @@ class ConsumerHost : public perfetto::Consumer, public mojom::ConsumerHost {
         DisableTracingAndEmitJsonCallback callback) override;
 
    private:
-    void OnJSONTraceData(std::string* json,
-                         base::DictionaryValue* metadata,
-                         bool has_more);
+    void ExportJson();
+    void OnJSONTraceData(std::string json, bool has_more);
     void OnEnableTracingTimeout();
     void MaybeSendEnableTracingAck();
     bool IsExpectedPid(base::ProcessId pid) const;
@@ -102,7 +102,9 @@ class ConsumerHost : public perfetto::Consumer, public mojom::ConsumerHost {
     bool privacy_filtering_enabled_ = false;
     base::SequenceBound<StreamWriter> read_buffers_stream_writer_;
     RequestBufferUsageCallback request_buffer_usage_callback_;
-    std::unique_ptr<JSONTraceExporter> json_trace_exporter_;
+    std::unique_ptr<perfetto::trace_processor::TraceProcessorStorage>
+        trace_processor_;
+    std::string json_agent_label_filter_;
     base::OnceCallback<void(bool)> flush_callback_;
     const mojom::TracingClientPriority tracing_priority_;
     base::OnceClosure on_disabled_callback_;

third_party/blink/web_tests/inspector-protocol/timeline/timeline-layout.js

@@ -21,23 +21,21 @@
   await tracingHelper.invokeAsyncWithTracing(performActions);
 
   var schedRecalc = tracingHelper.findEvent('ScheduleStyleRecalculation', 'I');
-  var recalcBegin = tracingHelper.findEvent('UpdateLayoutTree', 'B');
-  var recalcEnd = tracingHelper.findEvent('UpdateLayoutTree', 'E');
-  testRunner.log('UpdateLayoutTree frames match: ' + (schedRecalc.args.data.frame === recalcBegin.args.beginData.frame));
-  testRunner.log('UpdateLayoutTree elementCount > 0: ' + (recalcEnd.args.elementCount > 0));
+  var recalc = tracingHelper.findEvent('UpdateLayoutTree', 'X');
+  testRunner.log('UpdateLayoutTree frames match: ' + (schedRecalc.args.data.frame === recalc.args.beginData.frame));
+  testRunner.log('UpdateLayoutTree elementCount > 0: ' + (recalc.args.elementCount > 0));
 
   var invalidate = tracingHelper.findEvent('InvalidateLayout', 'I');
-  var layoutBegin = tracingHelper.findEvent('Layout', 'B');
-  var layoutEnd = tracingHelper.findEvent('Layout', 'E');
+  var layout = tracingHelper.findEvent('Layout', 'X');
 
-  testRunner.log('InvalidateLayout frames match: ' + (recalcBegin.args.beginData.frame === invalidate.args.data.frame));
+  testRunner.log('InvalidateLayout frames match: ' + (recalc.args.beginData.frame === invalidate.args.data.frame));
 
-  var beginData = layoutBegin.args.beginData;
+  var beginData = layout.args.beginData;
   testRunner.log('Layout frames match: ' + (invalidate.args.data.frame === beginData.frame));
   testRunner.log('dirtyObjects > 0: ' + (beginData.dirtyObjects > 0));
   testRunner.log('totalObjects > 0: ' + (beginData.totalObjects > 0));
 
-  var endData = layoutEnd.args.endData;
+  var endData = layout.args.endData;
   testRunner.log('has rootNode id: ' + (endData.rootNode > 0));
   testRunner.log('has root quad: ' + !!endData.root);