Reland "perfetto: Use TraceProcessor for proto2json conversion"

This is a reland of 0d929861.

Reland after fixes for crbug.com/1022770, crbug.com/1022386, and
crbug.com/1022818 landed.

Original change's description:
> perfetto: Use TraceProcessor for proto2json conversion
>
> TraceProcessor replaces the existing (unmaintainable) proto2json
> converter and adds features like combining of BEGIN/END events
> and support for heap profiles.
>
> Also updates a devtools test that was expecting separate begin/end
> events where TraceProcessor now combines them into a complete event.
>
> We have reduced the trace procesor's binary size overhead
> by selectively excluding features Chrome doesn't need from >400kB to
> ~50kB. We could probably save a couple more kB if we invest even more
> eng time and accept the maintenance burden for this in trace processor,
> however, we feel that we've reached an acceptable state that balances
> binary size & code health.
>
> Binary-Size: see paragraph above.
> Bug: 1006770
> Change-Id: Ia7f6b3a784c1a9c40574e1dcd80eca1ca2dfb03f
> Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1746258
> Commit-Queue: Eric Seckler <eseckler@chromium.org>
> Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
> Reviewed-by: oysteine <oysteine@chromium.org>
> Reviewed-by: Sami Kyöstilä <skyostil@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#713593}

Binary-Size: see paragraph above.
Bug: 1006770, 1022770, 1022386, 1022818
Change-Id: I7297721eb67834905262081282936013b34a0184
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1913602Reviewed-by: Sami Kyöstilä <skyostil@chromium.org>
Commit-Queue: Eric Seckler <eseckler@chromium.org>
Cr-Commit-Position: refs/heads/master@{#714887}

services/tracing/BUILD.gn

@@ -16,14 +16,10 @@ source_set("lib") {
   sources = [
     "perfetto/consumer_host.cc",
     "perfetto/consumer_host.h",
-    "perfetto/json_trace_exporter.cc",
-    "perfetto/json_trace_exporter.h",
     "perfetto/perfetto_service.cc",
     "perfetto/perfetto_service.h",
     "perfetto/producer_host.cc",
     "perfetto/producer_host.h",
-    "perfetto/track_event_json_exporter.cc",
-    "perfetto/track_event_json_exporter.h",
     "tracing_service.cc",
     "tracing_service.h",
   ]
@@ -40,31 +36,40 @@ source_set("lib") {
   ]
 
   deps = [
-    "//third_party/perfetto/protos/perfetto/trace/chrome:minimal_complete_lite",
+    "//third_party/perfetto:libproto_to_json",
   ]
 }
 
-executable("trace_json_exporter") {
+source_set("json_trace_exporter") {
   sources = [
-    "perfetto/json_exporter_main.cc",
     "perfetto/json_trace_exporter.cc",
     "perfetto/json_trace_exporter.h",
     "perfetto/track_event_json_exporter.cc",
     "perfetto/track_event_json_exporter.h",
   ]
-
-  configs += [ "//build/config/compiler:rtti" ]
-
   deps = [
     "//base",
     "//third_party/perfetto:libperfetto",
-    "//third_party/perfetto/include/perfetto/protozero:protozero",
     "//third_party/perfetto/protos/perfetto/common:lite",
     "//third_party/perfetto/protos/perfetto/trace:lite",
     "//third_party/perfetto/protos/perfetto/trace/chrome:lite",
     "//third_party/perfetto/protos/perfetto/trace/chrome:minimal_complete_lite",
     "//third_party/perfetto/protos/perfetto/trace/interned_data:lite",
     "//third_party/perfetto/protos/perfetto/trace/track_event:lite",
+  ]
+}
+
+executable("trace_json_exporter") {
+  sources = [
+    "perfetto/json_exporter_main.cc",
+  ]
+
+  configs += [ "//build/config/compiler:rtti" ]
+
+  deps = [
+    ":json_trace_exporter",
+    "//base",
+    "//third_party/perfetto:libperfetto",
     "//third_party/perfetto/src/protozero:protozero",
   ]
 }
@@ -145,6 +150,7 @@ source_set("tests") {
   }
 
   deps = [
+    ":json_trace_exporter",
     ":lib",
     ":test_utils",
     "//base",

services/tracing/perfetto/consumer_host.cc

@@ -24,13 +24,15 @@
 #include "mojo/public/cpp/bindings/self_owned_receiver.h"
 #include "mojo/public/cpp/bindings/strong_binding.h"
 #include "mojo/public/cpp/system/wait.h"
-#include "services/tracing/perfetto/json_trace_exporter.h"
 #include "services/tracing/perfetto/perfetto_service.h"
-#include "services/tracing/perfetto/track_event_json_exporter.h"
 #include "services/tracing/public/cpp/trace_event_args_whitelist.h"
+#include "third_party/perfetto/include/perfetto/ext/trace_processor/export_json.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/observable_events.h"
+#include "third_party/perfetto/include/perfetto/ext/tracing/core/slice.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/trace_packet.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/trace_stats.h"
+#include "third_party/perfetto/include/perfetto/trace_processor/basic_types.h"
+#include "third_party/perfetto/include/perfetto/trace_processor/trace_processor_storage.h"
 #include "third_party/perfetto/include/perfetto/tracing/core/trace_config.h"
 #include "third_party/perfetto/protos/perfetto/config/trace_config.pb.h"
 
@@ -40,6 +42,44 @@ namespace {
 
 const int32_t kEnableTracingTimeoutSeconds = 10;
 
+class JsonStringOutputWriter
+    : public perfetto::trace_processor::json::OutputWriter {
+ public:
+  using FlushCallback =
+      base::RepeatingCallback<void(std::string json, bool has_more)>;
+
+  JsonStringOutputWriter(FlushCallback flush_callback)
+      : flush_callback_(std::move(flush_callback)) {
+    buffer_.reserve(kBufferReserveCapacity);
+  }
+
+  ~JsonStringOutputWriter() override {
+    flush_callback_.Run(std::move(buffer_), false);
+  }
+
+  perfetto::trace_processor::util::Status AppendString(
+      const std::string& string) override {
+    buffer_ += string;
+    if (buffer_.size() > kBufferLimitInBytes) {
+      flush_callback_.Run(std::move(buffer_), true);
+      // Reset the buffer_ after moving it above.
+      buffer_.clear();
+      buffer_.reserve(kBufferReserveCapacity);
+    }
+    return perfetto::trace_processor::util::OkStatus();
+  }
+
+ private:
+  static constexpr size_t kBufferLimitInBytes = 100 * 1024;
+  // Since we write each string before checking the limit, we'll always go
+  // slightly over and hence we reserve some extra space to avoid most
+  // reallocs.
+  static constexpr size_t kBufferReserveCapacity = kBufferLimitInBytes * 5 / 4;
+
+  FlushCallback flush_callback_;
+  std::string buffer_;
+};
+
 }  // namespace
 
 class ConsumerHost::StreamWriter {
@@ -291,7 +331,7 @@ void ConsumerHost::TracingSession::OnTracingDisabled() {
 
   tracing_session_client_->OnTracingDisabled();
 
-  if (json_trace_exporter_) {
+  if (trace_processor_) {
     host_->consumer_endpoint()->ReadBuffers();
   }
 
@@ -343,6 +383,24 @@ void ConsumerHost::TracingSession::DisableTracingAndEmitJson(
                      weak_factory_.GetWeakPtr()),
       base::SequencedTaskRunnerHandle::Get());
 
+  if (privacy_filtering_enabled) {
+    // For filtering/whitelisting to be possible at JSON export time,
+    // filtering must not have been enabled during proto emission time
+    // (or there's nothing to pass through the whitelist).
+    DCHECK(!privacy_filtering_enabled_);
+    privacy_filtering_enabled_ = true;
+  }
+
+  json_agent_label_filter_ = agent_label_filter;
+
+  trace_processor_ =
+      perfetto::trace_processor::TraceProcessorStorage::CreateInstance(
+          perfetto::trace_processor::Config());
+
+  DisableTracing();
+}
+
+void ConsumerHost::TracingSession::ExportJson() {
   // In legacy backend, the trace event agent sets the predicate used by
   // TraceLog. For perfetto backend, ensure that predicate is always set
   // before creating the exporter. The agent can be created later than this
@@ -356,34 +414,55 @@ void ConsumerHost::TracingSession::DisableTracingAndEmitJson(
         base::BindRepeating(&IsMetadataWhitelisted));
   }
 
-  JSONTraceExporter::ArgumentFilterPredicate arg_filter_predicate;
-  JSONTraceExporter::MetadataFilterPredicate metadata_filter_predicate;
-  if (privacy_filtering_enabled) {
-    // For filtering/whitelisting to be possible at JSON export time,
-    // filtering must not have been enabled during proto emission time
-    // (or there's nothing to pass through the whitelist).
-    DCHECK(!privacy_filtering_enabled_);
+  perfetto::trace_processor::json::ArgumentFilterPredicate argument_filter;
+  perfetto::trace_processor::json::MetadataFilterPredicate metadata_filter;
+  perfetto::trace_processor::json::LabelFilterPredicate label_filter;
+
+  if (privacy_filtering_enabled_) {
     auto* trace_log = base::trace_event::TraceLog::GetInstance();
-    arg_filter_predicate = trace_log->GetArgumentFilterPredicate();
-    metadata_filter_predicate = trace_log->GetMetadataFilterPredicate();
+    base::trace_event::ArgumentFilterPredicate argument_filter_predicate =
+        trace_log->GetArgumentFilterPredicate();
+    argument_filter =
+        [argument_filter_predicate](
+            const char* category_group_name, const char* event_name,
+            perfetto::trace_processor::json::ArgumentNameFilterPredicate*
+                name_filter) {
+          base::trace_event::ArgumentNameFilterPredicate name_filter_predicate;
+          bool result = argument_filter_predicate.Run(
+              category_group_name, event_name, &name_filter_predicate);
+          if (name_filter_predicate) {
+            *name_filter = [name_filter_predicate](const char* arg_name) {
+              return name_filter_predicate.Run(arg_name);
+            };
+          }
+          return result;
+        };
+    base::trace_event::MetadataFilterPredicate metadata_filter_predicate =
+        trace_log->GetMetadataFilterPredicate();
+    metadata_filter = [metadata_filter_predicate](const char* metadata_name) {
+      return metadata_filter_predicate.Run(metadata_name);
+    };
   }
-  json_trace_exporter_ = std::make_unique<TrackEventJSONExporter>(
-      std::move(arg_filter_predicate), std::move(metadata_filter_predicate),
-      base::BindRepeating(&ConsumerHost::TracingSession::OnJSONTraceData,
-                          base::Unretained(this)));
 
-  json_trace_exporter_->set_label_filter(agent_label_filter);
+  if (!json_agent_label_filter_.empty()) {
+    label_filter = [this](const char* label) {
+      return strcmp(label, json_agent_label_filter_.c_str()) == 0;
+    };
+  }
 
-  DisableTracing();
+  JsonStringOutputWriter output_writer(base::BindRepeating(
+      &ConsumerHost::TracingSession::OnJSONTraceData, base::Unretained(this)));
+  auto status = perfetto::trace_processor::json::ExportJson(
+      trace_processor_.get(), &output_writer, argument_filter, metadata_filter,
+      label_filter);
+  DCHECK(status.ok()) << status.message();
 }
 
-void ConsumerHost::TracingSession::OnJSONTraceData(
-    std::string* json,
-    base::DictionaryValue* metadata,
-    bool has_more) {
+void ConsumerHost::TracingSession::OnJSONTraceData(std::string json,
+                                                   bool has_more) {
   auto slices = std::make_unique<StreamWriter::Slices>();
   slices->push_back(std::string());
-  slices->back().swap(*json);
+  slices->back().swap(json);
   read_buffers_stream_writer_.Post(FROM_HERE, &StreamWriter::WriteToStream,
                                    std::move(slices), has_more);
 
@@ -396,10 +475,40 @@ void ConsumerHost::TracingSession::OnTraceData(
     std::vector<perfetto::TracePacket> packets,
     bool has_more) {
   DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
-  if (json_trace_exporter_) {
-    json_trace_exporter_->OnTraceData(std::move(packets), has_more);
+  if (trace_processor_) {
+    // Calculate space needed for trace chunk. Each packet has a preamble and
+    // payload size.
+    size_t max_size = packets.size() * perfetto::TracePacket::kMaxPreambleBytes;
+    for (const auto& packet : packets) {
+      max_size += packet.size();
+    }
+
+    // Copy packets into a trace file chunk.
+    size_t position = 0;
+    std::unique_ptr<uint8_t[]> data(new uint8_t[max_size]);
+    for (perfetto::TracePacket& packet : packets) {
+      char* preamble;
+      size_t preamble_size;
+      std::tie(preamble, preamble_size) = packet.GetProtoPreamble();
+      DCHECK_LT(position + preamble_size, max_size);
+      memcpy(&data[position], preamble, preamble_size);
+      position += preamble_size;
+      for (const perfetto::Slice& slice : packet.slices()) {
+        DCHECK_LT(position + slice.size, max_size);
+        memcpy(&data[position], slice.start, slice.size);
+        position += slice.size;
+      }
+    }
+
+    auto status = trace_processor_->Parse(std::move(data), position);
+    // TODO(eseckler): There's no way to propagate this error at the moment - If
+    // one occurs on production builds, we silently ignore it and will end up
+    // producing an empty JSON result.
+    DCHECK(status.ok()) << status.message();
     if (!has_more) {
-      json_trace_exporter_.reset();
+      trace_processor_->NotifyEndOfFile();
+      ExportJson();
+      trace_processor_.reset();
     }
     return;
   }

services/tracing/perfetto/consumer_host.h

@@ -23,17 +23,18 @@
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/consumer.h"
 #include "third_party/perfetto/include/perfetto/ext/tracing/core/tracing_service.h"
 
-namespace base {
-class DictionaryValue;
-}
-
 namespace service_manager {
 struct BindSourceInfo;
 }  // namespace service_manager
 
+namespace perfetto {
+namespace trace_processor {
+class TraceProcessorStorage;
+}  // namespace trace_processor
+}  // namespace perfetto
+
 namespace tracing {
 
-class JSONTraceExporter;
 class PerfettoService;
 
 // This is a Mojo interface which enables any client
@@ -89,9 +90,8 @@ class ConsumerHost : public perfetto::Consumer, public mojom::ConsumerHost {
         DisableTracingAndEmitJsonCallback callback) override;
 
    private:
-    void OnJSONTraceData(std::string* json,
-                         base::DictionaryValue* metadata,
-                         bool has_more);
+    void ExportJson();
+    void OnJSONTraceData(std::string json, bool has_more);
     void OnEnableTracingTimeout();
     void MaybeSendEnableTracingAck();
     bool IsExpectedPid(base::ProcessId pid) const;
@@ -102,7 +102,9 @@ class ConsumerHost : public perfetto::Consumer, public mojom::ConsumerHost {
     bool privacy_filtering_enabled_ = false;
     base::SequenceBound<StreamWriter> read_buffers_stream_writer_;
     RequestBufferUsageCallback request_buffer_usage_callback_;
-    std::unique_ptr<JSONTraceExporter> json_trace_exporter_;
+    std::unique_ptr<perfetto::trace_processor::TraceProcessorStorage>
+        trace_processor_;
+    std::string json_agent_label_filter_;
     base::OnceCallback<void(bool)> flush_callback_;
     const mojom::TracingClientPriority tracing_priority_;
     base::OnceClosure on_disabled_callback_;

third_party/blink/web_tests/inspector-protocol/timeline/timeline-layout.js

@@ -21,23 +21,21 @@
   await tracingHelper.invokeAsyncWithTracing(performActions);
 
   var schedRecalc = tracingHelper.findEvent('ScheduleStyleRecalculation', 'I');
-  var recalcBegin = tracingHelper.findEvent('UpdateLayoutTree', 'B');
-  var recalcEnd = tracingHelper.findEvent('UpdateLayoutTree', 'E');
-  testRunner.log('UpdateLayoutTree frames match: ' + (schedRecalc.args.data.frame === recalcBegin.args.beginData.frame));
-  testRunner.log('UpdateLayoutTree elementCount > 0: ' + (recalcEnd.args.elementCount > 0));
+  var recalc = tracingHelper.findEvent('UpdateLayoutTree', 'X');
+  testRunner.log('UpdateLayoutTree frames match: ' + (schedRecalc.args.data.frame === recalc.args.beginData.frame));
+  testRunner.log('UpdateLayoutTree elementCount > 0: ' + (recalc.args.elementCount > 0));
 
   var invalidate = tracingHelper.findEvent('InvalidateLayout', 'I');
-  var layoutBegin = tracingHelper.findEvent('Layout', 'B');
-  var layoutEnd = tracingHelper.findEvent('Layout', 'E');
+  var layout = tracingHelper.findEvent('Layout', 'X');
 
-  testRunner.log('InvalidateLayout frames match: ' + (recalcBegin.args.beginData.frame === invalidate.args.data.frame));
+  testRunner.log('InvalidateLayout frames match: ' + (recalc.args.beginData.frame === invalidate.args.data.frame));
 
-  var beginData = layoutBegin.args.beginData;
+  var beginData = layout.args.beginData;
   testRunner.log('Layout frames match: ' + (invalidate.args.data.frame === beginData.frame));
   testRunner.log('dirtyObjects > 0: ' + (beginData.dirtyObjects > 0));
   testRunner.log('totalObjects > 0: ' + (beginData.totalObjects > 0));
 
-  var endData = layoutEnd.args.endData;
+  var endData = layout.args.endData;
   testRunner.log('has rootNode id: ' + (endData.rootNode > 0));
   testRunner.log('has root quad: ' + !!endData.root);