devtools: tracing: Add support for compressing the trace

Tracing can generate large output files (over 50M) especially if system
tracing is enabled, and are very compressible. Add an option to compress
them using the same machinery that chrome://tracing uses.

This also uses less /tmp space to store the stream. Storing large files
in /tmp was causing problems for chromecast, as space is /tmp is very
limited.

BUG=791708

Change-Id: Icbaac91cccb825a305431569bf6f42011d3cfbea
Reviewed-on: https://chromium-review.googlesource.com/807384
Commit-Queue: Michael Spang <spang@chromium.org>
Reviewed-by: Andrey Kosyakov <caseq@chromium.org>
Reviewed-by: Dmitry Gozman <dgozman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#521939}

content/browser/devtools/devtools_io_context.cc

@@ -48,7 +48,7 @@ unsigned s_last_stream_handle = 0;
 
 class TempFileStream : public DevToolsIOContext::RWStream {
  public:
-  TempFileStream();
+  explicit TempFileStream(bool binary);
 
   void Read(off_t position, size_t max_size, ReadCallback callback) override;
   void Close(bool invoke_pending_callbacks) override {}
@@ -67,16 +67,18 @@ class TempFileStream : public DevToolsIOContext::RWStream {
   scoped_refptr<base::SequencedTaskRunner> task_runner_;
   bool had_errors_;
   off_t last_read_pos_;
+  bool binary_;
 
   DISALLOW_COPY_AND_ASSIGN(TempFileStream);
 };
 
-TempFileStream::TempFileStream()
+TempFileStream::TempFileStream(bool binary)
     : DevToolsIOContext::RWStream(impl_task_runner()),
       handle_(base::UintToString(++s_last_stream_handle)),
       task_runner_(impl_task_runner()),
       had_errors_(false),
-      last_read_pos_(0) {}
+      last_read_pos_(0),
+      binary_(binary) {}
 
 TempFileStream::~TempFileStream() {
   DCHECK(task_runner_->RunsTasksInCurrentSequence());
@@ -129,6 +131,7 @@ void TempFileStream::ReadOnFileSequence(off_t position,
   DCHECK(task_runner_->RunsTasksInCurrentSequence());
   Status status = StatusFailure;
   std::unique_ptr<std::string> data;
+  bool base64_encoded = false;
 
   if (file_.IsValid()) {
     std::string buffer;
@@ -154,9 +157,14 @@ void TempFileStream::ReadOnFileSequence(off_t position,
       last_read_pos_ = position + size_got;
     }
   }
-  BrowserThread::PostTask(
-      BrowserThread::UI, FROM_HERE,
-      base::BindOnce(std::move(callback), std::move(data), false, status));
+  if (binary_) {
+    std::string raw_data(std::move(*data));
+    base::Base64Encode(raw_data, data.get());
+    base64_encoded = true;
+  }
+  BrowserThread::PostTask(BrowserThread::UI, FROM_HERE,
+                          base::BindOnce(std::move(callback), std::move(data),
+                                         base64_encoded, status));
 }
 
 void TempFileStream::AppendOnFileSequence(std::unique_ptr<std::string> data) {
@@ -472,8 +480,8 @@ DevToolsIOContext::~DevToolsIOContext() {
 }
 
 scoped_refptr<DevToolsIOContext::RWStream>
-DevToolsIOContext::CreateTempFileBackedStream() {
-  scoped_refptr<TempFileStream> result = new TempFileStream();
+DevToolsIOContext::CreateTempFileBackedStream(bool binary) {
+  scoped_refptr<TempFileStream> result = new TempFileStream(binary);
   bool inserted =
       streams_.insert(std::make_pair(result->handle(), result)).second;
   DCHECK(inserted);

content/browser/devtools/devtools_io_context.h

@@ -67,7 +67,7 @@ class DevToolsIOContext {
   DevToolsIOContext();
   ~DevToolsIOContext();
 
-  scoped_refptr<RWStream> CreateTempFileBackedStream();
+  scoped_refptr<RWStream> CreateTempFileBackedStream(bool binary);
   scoped_refptr<ROStream> GetByHandle(const std::string& handle);
   scoped_refptr<ROStream> OpenBlob(ChromeBlobStorageContext*,
                                    StoragePartition*,

content/browser/devtools/protocol/tracing_handler.cc

@@ -27,6 +27,7 @@
 #include "content/browser/devtools/devtools_io_context.h"
 #include "content/browser/devtools/devtools_session.h"
 #include "content/browser/tracing/tracing_controller_impl.h"
+#include "content/public/browser/browser_thread.h"
 #include "services/resource_coordinator/public/cpp/memory_instrumentation/memory_instrumentation.h"
 #include "services/resource_coordinator/public/interfaces/tracing/tracing_constants.mojom.h"
 
@@ -107,11 +108,26 @@ class DevToolsStreamEndpoint : public TracingController::TraceDataEndpoint {
       : stream_(stream), tracing_handler_(handler) {}
 
   void ReceiveTraceChunk(std::unique_ptr<std::string> chunk) override {
+    if (!BrowserThread::CurrentlyOn(BrowserThread::UI)) {
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(&DevToolsStreamEndpoint::ReceiveTraceChunk, this,
+                         std::move(chunk)));
+      return;
+    }
+
     stream_->Append(std::move(chunk));
   }
 
   void ReceiveTraceFinalContents(
       std::unique_ptr<const base::DictionaryValue> metadata) override {
+    if (!BrowserThread::CurrentlyOn(BrowserThread::UI)) {
+      BrowserThread::PostTask(
+          BrowserThread::UI, FROM_HERE,
+          base::BindOnce(&DevToolsStreamEndpoint::ReceiveTraceFinalContents,
+                         this, std::move(metadata)));
+      return;
+    }
     if (TracingHandler* h = tracing_handler_.get())
       h->OnTraceToStreamComplete(stream_->handle());
   }
@@ -134,6 +150,7 @@ TracingHandler::TracingHandler(TracingHandler::Target target,
       frame_tree_node_id_(frame_tree_node_id),
       did_initiate_recording_(false),
       return_as_stream_(false),
+      gzip_compression_(false),
       weak_factory_(this) {}
 
 TracingHandler::~TracingHandler() {
@@ -249,23 +266,30 @@ std::string TracingHandler::UpdateTraceDataBuffer(
 }
 
 void TracingHandler::OnTraceToStreamComplete(const std::string& stream_handle) {
-  frontend_->TracingComplete(stream_handle);
+  std::string stream_compression =
+      (gzip_compression_ ? Tracing::StreamCompressionEnum::Gzip
+                         : Tracing::StreamCompressionEnum::None);
+  frontend_->TracingComplete(stream_handle, stream_compression);
 }
 
 void TracingHandler::Start(Maybe<std::string> categories,
                            Maybe<std::string> options,
                            Maybe<double> buffer_usage_reporting_interval,
                            Maybe<std::string> transfer_mode,
+                           Maybe<std::string> transfer_compression,
                            Maybe<Tracing::TraceConfig> config,
                            std::unique_ptr<StartCallback> callback) {
   bool return_as_stream = transfer_mode.fromMaybe("") ==
       Tracing::Start::TransferModeEnum::ReturnAsStream;
+  bool gzip_compression = transfer_compression.fromMaybe("") ==
+                          Tracing::StreamCompressionEnum::Gzip;
   if (IsTracing()) {
     if (!did_initiate_recording_ && IsStartupTracingActive()) {
       // If tracing is already running because it was initiated by startup
       // tracing, honor the transfer mode update, as that's the only way
       // for the client to communicate it.
       return_as_stream_ = return_as_stream;
+      gzip_compression_ = gzip_compression;
     }
     callback->sendFailure(Response::Error("Tracing is already started"));
     return;
@@ -280,6 +304,7 @@ void TracingHandler::Start(Maybe<std::string> categories,
 
   did_initiate_recording_ = true;
   return_as_stream_ = return_as_stream;
+  gzip_compression_ = gzip_compression;
   if (buffer_usage_reporting_interval.isJust())
     SetupTimer(buffer_usage_reporting_interval.fromJust());
 
@@ -320,7 +345,12 @@ void TracingHandler::End(std::unique_ptr<EndCallback> callback) {
   scoped_refptr<TracingController::TraceDataEndpoint> endpoint;
   if (return_as_stream_) {
     endpoint = new DevToolsStreamEndpoint(
-        weak_factory_.GetWeakPtr(), io_context_->CreateTempFileBackedStream());
+        weak_factory_.GetWeakPtr(), io_context_->CreateTempFileBackedStream(
+                                        gzip_compression_ /* binary */));
+    if (gzip_compression_) {
+      endpoint = TracingControllerImpl::CreateCompressedStringEndpoint(
+          endpoint, true /* compress_with_background_priority */);
+    }
     StopTracing(endpoint, "");
   } else {
     // Reset the trace data buffer state.

content/browser/devtools/protocol/tracing_handler.h

@@ -55,6 +55,7 @@ class TracingHandler : public DevToolsDomainHandler, public Tracing::Backend {
              Maybe<std::string> options,
              Maybe<double> buffer_usage_reporting_interval,
              Maybe<std::string> transfer_mode,
+             Maybe<std::string> transfer_compression,
              Maybe<Tracing::TraceConfig> config,
              std::unique_ptr<StartCallback> callback) override;
   void End(std::unique_ptr<EndCallback> callback) override;
@@ -110,6 +111,7 @@ class TracingHandler : public DevToolsDomainHandler, public Tracing::Backend {
   int frame_tree_node_id_;
   bool did_initiate_recording_;
   bool return_as_stream_;
+  bool gzip_compression_;
   TraceDataBufferState trace_data_buffer_state_;
   base::WeakPtrFactory<TracingHandler> weak_factory_;
 

third_party/WebKit/Source/core/inspector/InspectorTracingAgent.cpp

@@ -62,6 +62,7 @@ void InspectorTracingAgent::start(Maybe<String> categories,
                                   Maybe<String> options,
                                   Maybe<double> buffer_usage_reporting_interval,
                                   Maybe<String> transfer_mode,
+                                  Maybe<String> transfer_compression,
                                   Maybe<protocol::Tracing::TraceConfig> config,
                                   std::unique_ptr<StartCallback> callback) {
   DCHECK(!IsStarted());

third_party/WebKit/Source/core/inspector/InspectorTracingAgent.h

@@ -53,6 +53,7 @@ class CORE_EXPORT InspectorTracingAgent final
              protocol::Maybe<String> options,
              protocol::Maybe<double> buffer_usage_reporting_interval,
              protocol::Maybe<String> transfer_mode,
+             protocol::Maybe<String> transfer_compression,
              protocol::Maybe<protocol::Tracing::TraceConfig>,
              std::unique_ptr<StartCallback>) override;
   void end(std::unique_ptr<EndCallback>) override;

third_party/WebKit/Source/core/inspector/browser_protocol.json

@@ -11866,6 +11866,15 @@
                             "$ref": "MemoryDumpConfig"
                         }
                     ]
+                },
+                {
+                    "id": "StreamCompression",
+                    "description": "Compression type to use for traces returned via streams.",
+                    "type": "string",
+                    "enum": [
+                        "none",
+                        "gzip"
+                    ]
                 }
             ],
             "commands": [
@@ -11948,6 +11957,12 @@
                                 "ReturnAsStream"
                             ]
                         },
+                        {
+                            "name": "streamCompression",
+                            "description": "Compression format to use. This only applies when using `ReturnAsStream` transfer mode (defaults to `none`)",
+                            "optional": true,
+                            "$ref": "StreamCompression"
+                        },
                         {
                             "name": "traceConfig",
                             "optional": true,
@@ -12002,6 +12017,12 @@
                             "description": "A handle of the stream that holds resulting trace data.",
                             "optional": true,
                             "$ref": "IO.StreamHandle"
+                        },
+                        {
+                            "name": "streamCompression",
+                            "description": "Compression format of returned stream.",
+                            "optional": true,
+                            "$ref": "StreamCompression"
                         }
                     ]
                 }

third_party/WebKit/Source/core/inspector/browser_protocol.pdl

@@ -5432,6 +5432,12 @@ experimental domain Tracing
       # Configuration for memory dump triggers. Used only when "memory-infra" category is enabled.
       optional MemoryDumpConfig memoryDumpConfig
 
+  # Compression type to use for traces returned via streams.
+  type StreamCompression extends string
+    enum
+      none
+      gzip
+
   # Stop trace events collection.
   command end
 
@@ -5469,6 +5475,9 @@ experimental domain Tracing
       optional enum transferMode
         ReportEvents
         ReturnAsStream
+      # Compression format to use. This only applies when using `ReturnAsStream`
+      # transfer mode (defaults to `none`)
+      optional StreamCompression streamCompression
       optional TraceConfig traceConfig
 
   event bufferUsage
@@ -5494,3 +5503,5 @@ experimental domain Tracing
     parameters
       # A handle of the stream that holds resulting trace data.
       optional IO.StreamHandle stream
+      # Compression format of returned stream.
+      optional StreamCompression streamCompression