Allow GZIP compression of remote-bound WebRTC event logs

Compress remote-bound WebRTC event logs while writing them to disk.
* Logs for which the file size budget did not allow covering the
  entire call, can now get more coverage.
* Short calls, for which we could log the entire call anyhow,
  benefit from a lower cost in bandwidth when the log is uploaded.

Compression takes place incrementally over the duration of the
call, avoiding a spike in resource utilization.

The file-size limit is very strictly abided by. Before compression
takes place, an estimation is made over whether the compressed
string will fit in the file without exceeding the budget. If not,
the file is terminated (as it would for the uncompressed version).

In the unlikely case that the estimation is mistaken, and we've
compressed to the point that writing to the file would make it
exceed its allotted maximum size, the file is dicarded. (It is
infeasible to write a compression footer at that point.)

Bug: 775415
Change-Id: Iab6b91815c4e9e855ced16668dbfd65ed64ad9f0
Reviewed-on: https://chromium-review.googlesource.com/1142147
Commit-Queue: Elad Alon <eladalon@chromium.org>
Reviewed-by: Guido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#580287}

chrome/browser/BUILD.gn

@@ -1640,7 +1640,7 @@ jumbo_split_static_library("browser") {
     "//build/config/compiler:wexit_time_destructors",
     "//build/config:precompiled_headers",
   ]
-  defines = []
+  defines = [ "ZLIB_CONST" ]
   libs = []
   ldflags = []
 

chrome/browser/media/webrtc/webrtc_event_log_manager.cc

@@ -392,6 +392,11 @@ WebRtcEventLogManager::CreateRemoteLogFileWriterFactory() {
 #else
   if (remote_log_file_writer_factory_for_testing_) {
     return std::move(remote_log_file_writer_factory_for_testing_);
+  } else if (base::FeatureList::IsEnabled(
+                 features::kWebRtcRemoteEventLogGzipped)) {
+    return std::make_unique<GzippedLogFileWriterFactory>(
+        std::make_unique<GzipLogCompressorFactory>(
+            std::make_unique<DefaultGzippedSizeEstimator::Factory>()));
   } else {
     return std::make_unique<BaseLogFileWriterFactory>();
   }

chrome/browser/media/webrtc/webrtc_event_log_manager_common.h

@@ -43,16 +43,19 @@ extern const size_t kMaxActiveRemoteBoundWebRtcEventLogs;
 // limit is applied per browser context.
 extern const size_t kMaxPendingRemoteBoundWebRtcEventLogs;
 
+// Overhead incurred by GZIP due to its header and footer.
+extern const size_t kGzipOverheadBytes;
+
 // Remote-bound log files' names will be of the format [prefix]_[log_id].[ext],
 // where |prefix| is equal to kRemoteBoundWebRtcEventLogFileNamePrefix,
 // |log_id| is composed of 32 random characters from '0'-'9' and 'A'-'F',
 // and |ext| is the extension determined by the used LogCompressor::Factory,
 // which will be either kWebRtcEventLogUncompressedExtension or
 // kWebRtcEventLogGzippedExtension.
-// TODO(crbug.com/775415): Add kWebRtcEventLogGzippedExtension.
 extern const base::FilePath::CharType
     kRemoteBoundWebRtcEventLogFileNamePrefix[];
 extern const base::FilePath::CharType kWebRtcEventLogUncompressedExtension[];
+extern const base::FilePath::CharType kWebRtcEventLogGzippedExtension[];
 
 // Remote-bound event logs will not be uploaded if the time since their last
 // modification (meaning the time when they were completed) exceeds this value.
@@ -258,6 +261,148 @@ class BaseLogFileWriterFactory : public LogFileWriter::Factory {
       base::Optional<size_t> max_file_size_bytes) const override;
 };
 
+// Interface for a class that provides compression of a stream, while attempting
+// to observe a limit on the size.
+//
+// One should note that:
+// * For compressors that use a footer, to guarantee proper decompression,
+//   the footer must be written to the file.
+// * In such a case, usually, nothing can be omitted from the file, or the
+//   footer's CRC (if used) would be wrong.
+// * Determining a string's size pre-compression, without performing the actual
+//   compression, is heuristic in nature.
+//
+// Therefore, compression might terminate (FULL) earlier than it
+// must, or even in theory (which we attempt to avoid in practice) exceed the
+// size allowed it, in which case the file will be discarded (ERROR).
+class LogCompressor {
+ public:
+  // By subclassing this factory, concrete implementations of LogCompressor can
+  // be produced by unit tests, while keeping their definition in the .cc file.
+  // (Only the factory needs to be declared in the header.)
+  class Factory {
+   public:
+    virtual ~Factory() = default;
+
+    // The smallest size a log file of this type may assume.
+    virtual size_t MinSizeBytes() const = 0;
+
+    // Returns a LogCompressor if the parameters are valid and all
+    // initializations are successful; en empty unique_ptr otherwise.
+    // If !max_size_bytes.has_value(), an unlimited compressor is created.
+    virtual std::unique_ptr<LogCompressor> Create(
+        base::Optional<size_t> max_size_bytes) const = 0;
+  };
+
+  // Result of a call to Compress().
+  // * OK and ERROR_ENCOUNTERED are self-explanatory.
+  // * DISALLOWED means that, due to budget constraints, the input could
+  //   not be compressed. The stream is still in a legal state, but only
+  //   a call to CreateFooter() is now allowed.
+  enum class Result { OK, DISALLOWED, ERROR_ENCOUNTERED };
+
+  virtual ~LogCompressor() = default;
+
+  // Produces a compression header and writes it to |output|.
+  // The size does not count towards the max size limit.
+  // Guaranteed not to fail (nothing can realistically go wrong).
+  virtual void CreateHeader(std::string* output) = 0;
+
+  // Compresses |input| into |output|.
+  // * If compression succeeded, and the budget was observed, OK is returned.
+  // * If the compressor thinks the string, once compressed, will exceed the
+  //   maximum size (when combined with previously compressed strings),
+  //   compression will not be done, and DISALLOWED will be returned.
+  //   This allows producing a valid footer without exceeding the size limit.
+  // * Unexpected errors in the underlying compressor (e.g. zlib, etc.),
+  //   or unexpectedly getting a compressed string which exceeds the budget,
+  //   will return ERROR_ENCOUNTERED.
+  // This function may not be called again if DISALLOWED or ERROR_ENCOUNTERED
+  // were ever returned before, or after CreateFooter() was called.
+  virtual Result Compress(const std::string& input, std::string* output) = 0;
+
+  // Produces a compression footer and writes it to |output|.
+  // The footer does not count towards the max size limit.
+  // May not be called more than once, or if Compress() returned ERROR.
+  virtual bool CreateFooter(std::string* output) = 0;
+};
+
+// Estimates the compressed size, without performing compression (except in
+// unit tests, where performance is of lesser importance).
+// This interface allows unit tests to simulate specific cases, such as
+// over/under-estimation, and show that the code using the LogCompressor
+// deals with them correctly. (E.g., if the estimation expects the compression
+// to not go over-budget, but then it does.)
+// The estimator is expected to be stateful. That is, the order of calls to
+// EstimateCompressedSize() should correspond to the order of calls
+// to Compress().
+class CompressedSizeEstimator {
+ public:
+  class Factory {
+   public:
+    virtual ~Factory() = default;
+    virtual std::unique_ptr<CompressedSizeEstimator> Create() const = 0;
+  };
+
+  virtual ~CompressedSizeEstimator() = default;
+
+  virtual size_t EstimateCompressedSize(const std::string& input) const = 0;
+};
+
+// Provides a conservative estimation of the number of bytes required to
+// compress a string using GZIP. This estimation is not expected to ever
+// be overly optimistic, but the code using it should nevertheless be prepared
+// to deal with that theoretical possibility.
+class DefaultGzippedSizeEstimator : public CompressedSizeEstimator {
+ public:
+  class Factory : public CompressedSizeEstimator::Factory {
+   public:
+    ~Factory() override = default;
+
+    std::unique_ptr<CompressedSizeEstimator> Create() const override;
+  };
+
+  ~DefaultGzippedSizeEstimator() override = default;
+
+  size_t EstimateCompressedSize(const std::string& input) const override;
+};
+
+// Interface for producing LogCompressorGzip objects.
+class GzipLogCompressorFactory : public LogCompressor::Factory {
+ public:
+  explicit GzipLogCompressorFactory(
+      std::unique_ptr<CompressedSizeEstimator::Factory> estimator_factory);
+  ~GzipLogCompressorFactory() override;
+
+  size_t MinSizeBytes() const override;
+
+  std::unique_ptr<LogCompressor> Create(
+      base::Optional<size_t> max_size_bytes) const override;
+
+ private:
+  std::unique_ptr<CompressedSizeEstimator::Factory> estimator_factory_;
+};
+
+// Produces LogFileWriter instances that perform compression using GZIP.
+class GzippedLogFileWriterFactory : public LogFileWriter::Factory {
+ public:
+  explicit GzippedLogFileWriterFactory(
+      std::unique_ptr<GzipLogCompressorFactory> gzip_compressor_factory);
+
+  ~GzippedLogFileWriterFactory() override;
+
+  size_t MinFileSizeBytes() const override;
+
+  base::FilePath::StringPieceType Extension() const override;
+
+  std::unique_ptr<LogFileWriter> Create(
+      const base::FilePath& path,
+      base::Optional<size_t> max_file_size_bytes) const override;
+
+ private:
+  std::unique_ptr<GzipLogCompressorFactory> gzip_compressor_factory_;
+};
+
 // Translate a BrowserContext into an ID. This lets us associate PeerConnections
 // with BrowserContexts, while making sure that we never call the
 // BrowserContext's methods outside of the UI thread (because we can't call them

chrome/browser/media/webrtc/webrtc_event_log_manager_remote.cc

@@ -537,15 +537,19 @@ void WebRtcRemoteEventLogManager::AddPendingLogs(
     const base::FilePath& remote_bound_logs_dir) {
   DCHECK(task_runner_->RunsTasksInCurrentSequence());
 
-  base::FilePath::StringType pattern =
-      base::FilePath(FILE_PATH_LITERAL("*"))
-          .AddExtension(log_file_writer_factory_->Extension())
-          .value();
   base::FileEnumerator enumerator(remote_bound_logs_dir,
                                   /*recursive=*/false,
-                                  base::FileEnumerator::FILES, pattern);
+                                  base::FileEnumerator::FILES);
 
   for (auto path = enumerator.Next(); !path.empty(); path = enumerator.Next()) {
+    const base::FileEnumerator::FileInfo info = enumerator.GetInfo();
+    const base::FilePath::StringType extension = info.GetName().Extension();
+    const auto separator =
+        base::FilePath::StringType(1, base::FilePath::kExtensionSeparator);
+    if (extension != separator + kWebRtcEventLogUncompressedExtension &&
+        extension != separator + kWebRtcEventLogGzippedExtension) {
+      continue;
+    }
     const auto last_modified = enumerator.GetInfo().GetLastModifiedTime();
     auto it = pending_logs_.emplace(browser_context_id, path, last_modified);
     DCHECK(it.second);  // No pre-existing entry.

chrome/browser/media/webrtc/webrtc_event_log_manager_remote.h

@@ -340,7 +340,6 @@ class WebRtcRemoteEventLogManager final
   std::map<PeerConnectionKey, const std::string> active_peer_connections_;
 
   // Creates LogFileWriter instances (compressed/uncompressed, etc.).
-  // TODO(crbug.com/775415): Add support for compressed version using GZIP.
   std::unique_ptr<LogFileWriter::Factory> log_file_writer_factory_;
 
   // Remote-bound logs which we're currently in the process of writing to disk.

chrome/browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.cc

@@ -14,8 +14,15 @@ std::unique_ptr<LogFileWriter::Factory> CreateLogFileWriterFactory(
   switch (compression) {
     case WebRtcEventLogCompression::NONE:
       return std::make_unique<BaseLogFileWriterFactory>();
+    case WebRtcEventLogCompression::GZIP_NULL_ESTIMATION:
+      return std::make_unique<GzippedLogFileWriterFactory>(
+          std::make_unique<GzipLogCompressorFactory>(
+              std::make_unique<NullEstimator::Factory>()));
+    case WebRtcEventLogCompression::GZIP_PERFECT_ESTIMATION:
+      return std::make_unique<GzippedLogFileWriterFactory>(
+          std::make_unique<GzipLogCompressorFactory>(
+              std::make_unique<PerfectGzipEstimator::Factory>()));
   }
-
   NOTREACHED();
   return nullptr;  // Appease compiler.
 }
@@ -31,3 +38,57 @@ void RemoveWritePermissions(const base::FilePath& path) {
   ASSERT_TRUE(base::SetPosixFilePermissions(path, permissions));
 }
 #endif  // defined(OS_POSIX)
+
+std::unique_ptr<CompressedSizeEstimator> NullEstimator::Factory::Create()
+    const {
+  return std::make_unique<NullEstimator>();
+}
+
+size_t NullEstimator::EstimateCompressedSize(const std::string& input) const {
+  return 0;
+}
+
+std::unique_ptr<CompressedSizeEstimator> PerfectGzipEstimator::Factory::Create()
+    const {
+  return std::make_unique<PerfectGzipEstimator>();
+}
+
+PerfectGzipEstimator::PerfectGzipEstimator() {
+  // This factory will produce an optimistic compressor that will always
+  // think it can compress additional inputs, which will therefore allow
+  // us to find out what the real compressed size it, since compression
+  // will never be suppressed.
+  GzipLogCompressorFactory factory(std::make_unique<NullEstimator::Factory>());
+
+  compressor_ = factory.Create(base::Optional<size_t>());
+  DCHECK(compressor_);
+
+  std::string ignored;
+  compressor_->CreateHeader(&ignored);
+}
+
+PerfectGzipEstimator::~PerfectGzipEstimator() = default;
+
+size_t PerfectGzipEstimator::EstimateCompressedSize(
+    const std::string& input) const {
+  std::string output;
+  EXPECT_EQ(compressor_->Compress(input, &output), LogCompressor::Result::OK);
+  return output.length();
+}
+
+size_t GzippedSize(const std::string& uncompressed) {
+  PerfectGzipEstimator perfect_estimator;
+  return kGzipOverheadBytes +
+         perfect_estimator.EstimateCompressedSize(uncompressed);
+}
+
+size_t GzippedSize(const std::vector<std::string>& uncompressed) {
+  PerfectGzipEstimator perfect_estimator;
+
+  size_t result = kGzipOverheadBytes;
+  for (const std::string& str : uncompressed) {
+    result += perfect_estimator.EstimateCompressedSize(str);
+  }
+
+  return result;
+}

chrome/browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.h

@@ -6,6 +6,7 @@
 #define CHROME_BROWSER_MEDIA_WEBRTC_WEBRTC_EVENT_LOG_MANAGER_UNITTEST_HELPERS_H_
 
 #include <memory>
+#include <string>
 
 #include "base/files/file_path.h"
 #include "build/build_config.h"
@@ -13,8 +14,9 @@
 
 // Which type of compression, if any, LogFileWriterTest should use.
 enum class WebRtcEventLogCompression {
-  NONE
-  // TODO(crbug.com/775415): Add support for GZIP.
+  NONE,
+  GZIP_NULL_ESTIMATION,
+  GZIP_PERFECT_ESTIMATION
 };
 
 // Produce a LogFileWriter::Factory object.
@@ -25,4 +27,52 @@ std::unique_ptr<LogFileWriter::Factory> CreateLogFileWriterFactory(
 void RemoveWritePermissions(const base::FilePath& path);
 #endif  // defined(OS_POSIX)
 
+// Always estimates strings to be compressed to zero bytes.
+class NullEstimator : public CompressedSizeEstimator {
+ public:
+  class Factory : public CompressedSizeEstimator::Factory {
+   public:
+    ~Factory() override = default;
+
+    std::unique_ptr<CompressedSizeEstimator> Create() const override;
+  };
+
+  ~NullEstimator() override = default;
+
+  size_t EstimateCompressedSize(const std::string& input) const override;
+};
+
+// Provides a perfect estimation of the compressed size by cheating - performing
+// actual compression, then reporting the resulting size.
+// This class is stateful; the number, nature and order of calls to
+// EstimateCompressedSize() is important.
+class PerfectGzipEstimator : public CompressedSizeEstimator {
+ public:
+  class Factory : public CompressedSizeEstimator::Factory {
+   public:
+    ~Factory() override = default;
+
+    std::unique_ptr<CompressedSizeEstimator> Create() const override;
+  };
+
+  PerfectGzipEstimator();
+
+  ~PerfectGzipEstimator() override;
+
+  size_t EstimateCompressedSize(const std::string& input) const override;
+
+ private:
+  // This compressor allows EstimateCompressedSize to return an exact estimate.
+  // EstimateCompressedSize is normally const, but here we fake it, so we set
+  // it as mutable.
+  mutable std::unique_ptr<LogCompressor> compressor_;
+};
+
+// Check the gzipped size of |uncompressed|, including header and footer,
+// assuming it were gzipped on its own.
+size_t GzippedSize(const std::string& uncompressed);
+
+// Same as other version, but with elements compressed in sequence.
+size_t GzippedSize(const std::vector<std::string>& uncompressed);
+
 #endif  // CHROME_BROWSER_MEDIA_WEBRTC_WEBRTC_EVENT_LOG_MANAGER_UNITTEST_HELPERS_H_

chrome/common/chrome_features.cc

@@ -655,6 +655,9 @@ const base::Feature kMachineLearningService{"MachineLearningService",
 // Please note that a Chrome policy must also be set, for this to have effect.
 extern const base::Feature kWebRtcRemoteEventLog{
     "WebRtcRemoteEventLog", base::FEATURE_DISABLED_BY_DEFAULT};
+// Compress remote-bound WebRTC event logs (if used; see kWebRtcRemoteEventLog).
+extern const base::Feature kWebRtcRemoteEventLogGzipped{
+    "WebRtcRemoteEventLogGzipped", base::FEATURE_ENABLED_BY_DEFAULT};
 #endif
 
 #if defined(OS_WIN)

chrome/common/chrome_features.h

@@ -355,6 +355,7 @@ extern const base::Feature kMachineLearningService;
 
 #if !defined(OS_ANDROID)
 extern const base::Feature kWebRtcRemoteEventLog;
+extern const base::Feature kWebRtcRemoteEventLogGzipped;
 #endif
 
 #if defined(OS_WIN)

chrome/test/BUILD.gn

@@ -2989,6 +2989,7 @@ test("unit_tests") {
       "../browser/media/router/discovery/discovery_network_monitor_metric_observer_unittest.cc",
       "../browser/media/router/discovery/discovery_network_monitor_unittest.cc",
       "../browser/media/webrtc/tab_desktop_media_list_unittest.cc",
+      "../browser/media/webrtc/webrtc_event_log_manager_common_unittest.cc",
       "../browser/media/webrtc/webrtc_event_log_manager_unittest.cc",
       "../browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.cc",
       "../browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.h",