Commit 02e70958 authored by Elad Alon's avatar Elad Alon Committed by Commit Bot

Allow GZIP compression of remote-bound WebRTC event logs

Compress remote-bound WebRTC event logs while writing them to disk.
* Logs for which the file size budget did not allow covering the
  entire call, can now get more coverage.
* Short calls, for which we could log the entire call anyhow,
  benefit from a lower cost in bandwidth when the log is uploaded.

Compression takes place incrementally over the duration of the
call, avoiding a spike in resource utilization.

The file-size limit is very strictly abided by. Before compression
takes place, an estimation is made over whether the compressed
string will fit in the file without exceeding the budget. If not,
the file is terminated (as it would for the uncompressed version).

In the unlikely case that the estimation is mistaken, and we've
compressed to the point that writing to the file would make it
exceed its allotted maximum size, the file is dicarded. (It is
infeasible to write a compression footer at that point.)

Bug: 775415
Change-Id: Iab6b91815c4e9e855ced16668dbfd65ed64ad9f0
Reviewed-on: https://chromium-review.googlesource.com/1142147
Commit-Queue: Elad Alon <eladalon@chromium.org>
Reviewed-by: default avatarGuido Urdaneta <guidou@chromium.org>
Cr-Commit-Position: refs/heads/master@{#580287}
parent 27518dd2
......@@ -1640,7 +1640,7 @@ jumbo_split_static_library("browser") {
"//build/config/compiler:wexit_time_destructors",
"//build/config:precompiled_headers",
]
defines = []
defines = [ "ZLIB_CONST" ]
libs = []
ldflags = []
......
......@@ -392,6 +392,11 @@ WebRtcEventLogManager::CreateRemoteLogFileWriterFactory() {
#else
if (remote_log_file_writer_factory_for_testing_) {
return std::move(remote_log_file_writer_factory_for_testing_);
} else if (base::FeatureList::IsEnabled(
features::kWebRtcRemoteEventLogGzipped)) {
return std::make_unique<GzippedLogFileWriterFactory>(
std::make_unique<GzipLogCompressorFactory>(
std::make_unique<DefaultGzippedSizeEstimator::Factory>()));
} else {
return std::make_unique<BaseLogFileWriterFactory>();
}
......
......@@ -13,6 +13,7 @@
#include "content/public/browser/browser_context.h"
#include "content/public/browser/browser_thread.h"
#include "content/public/browser/render_process_host.h"
#include "third_party/zlib/zlib.h"
using BrowserContextId = WebRtcEventLogPeerConnectionKey::BrowserContextId;
......@@ -34,6 +35,11 @@ const BrowserContextId kNullBrowserContextId =
namespace {
constexpr int kDefaultMemLevel = 8;
constexpr size_t kGzipHeaderBytes = 15;
constexpr size_t kGzipFooterBytes = 10;
// Tracks budget over a resource (such as bytes allowed in a file, etc.).
// Allows an unlimited budget.
class Budget {
......@@ -193,7 +199,6 @@ const base::FilePath& BaseLogFileWriter::path() const {
bool BaseLogFileWriter::MaxSizeReached() const {
DCHECK(task_runner_->RunsTasksInCurrentSequence());
// Well-behaved code wouldn't check otherwise.
DCHECK_EQ(state(), State::ACTIVE);
return !WithinBudget(1);
}
......@@ -212,11 +217,11 @@ bool BaseLogFileWriter::Write(const std::string& input) {
return false;
}
const bool written = WriteInternal(input, /*metadata=*/false);
if (!written) {
const bool did_write = WriteInternal(input, /*metadata=*/false);
if (!did_write) {
SetState(State::ERRORED);
}
return written;
return did_write;
}
bool BaseLogFileWriter::Close() {
......@@ -296,10 +301,385 @@ bool BaseLogFileWriter::Finalize() {
return true;
}
// Writes a GZIP-compressed log to a file while observing a maximum size.
class GzippedLogFileWriter : public BaseLogFileWriter {
public:
GzippedLogFileWriter(const base::FilePath& path,
base::Optional<size_t> max_file_size_bytes,
std::unique_ptr<LogCompressor> compressor);
~GzippedLogFileWriter() override = default;
bool Init() override;
bool MaxSizeReached() const override;
bool Write(const std::string& input) override;
protected:
bool Finalize() override;
private:
std::unique_ptr<LogCompressor> compressor_;
};
GzippedLogFileWriter::GzippedLogFileWriter(
const base::FilePath& path,
base::Optional<size_t> max_file_size_bytes,
std::unique_ptr<LogCompressor> compressor)
: BaseLogFileWriter(path, max_file_size_bytes),
compressor_(std::move(compressor)) {
// Factory validates size before instantiation.
DCHECK(!max_file_size_bytes.has_value() ||
max_file_size_bytes.value() >= kGzipOverheadBytes);
}
bool GzippedLogFileWriter::Init() {
if (!BaseLogFileWriter::Init()) {
// Super-class should SetState on its own.
return false;
}
std::string header;
compressor_->CreateHeader(&header);
const bool result = WriteInternal(header, /*metadata=*/true);
if (!result) {
SetState(State::ERRORED);
}
return result;
}
bool GzippedLogFileWriter::MaxSizeReached() const {
DCHECK_EQ(state(), State::ACTIVE);
// Note that the overhead used (footer only) assumes state() is State::ACTIVE,
// as DCHECKed above.
return !WithinBudget(1 + kGzipFooterBytes);
}
bool GzippedLogFileWriter::Write(const std::string& input) {
DCHECK_EQ(state(), State::ACTIVE);
DCHECK(!MaxSizeReached());
if (input.empty()) {
return true;
}
std::string compressed_input;
const auto result = compressor_->Compress(input, &compressed_input);
switch (result) {
case LogCompressor::Result::OK: {
// |compressor_| guarantees |compressed_input| is within-budget.
bool did_write = WriteInternal(compressed_input, /*metadata=*/false);
if (!did_write) {
SetState(State::ERRORED);
}
return did_write;
}
case LogCompressor::Result::DISALLOWED: {
SetState(State::FULL);
return false;
}
case LogCompressor::Result::ERROR_ENCOUNTERED: {
SetState(State::ERRORED);
return false;
}
}
NOTREACHED();
return false; // Appease compiler.
}
bool GzippedLogFileWriter::Finalize() {
DCHECK_NE(state(), State::CLOSED);
DCHECK_NE(state(), State::DELETED);
DCHECK_NE(state(), State::ERRORED);
std::string footer;
if (!compressor_->CreateFooter(&footer)) {
LOG(WARNING) << "Compression footer could not be produced.";
SetState(State::ERRORED);
return false;
}
// |compressor_| guarantees |footer| is within-budget.
if (!WriteInternal(footer, /*metadata=*/true)) {
LOG(WARNING) << "Footer could not be written.";
SetState(State::ERRORED);
return false;
}
return true;
}
// Concrete implementation of LogCompressor using GZIP.
class GzipLogCompressor : public LogCompressor {
public:
GzipLogCompressor(
base::Optional<size_t> max_size_bytes,
std::unique_ptr<CompressedSizeEstimator> compressed_size_estimator);
~GzipLogCompressor() override;
void CreateHeader(std::string* output) override;
Result Compress(const std::string& input, std::string* output) override;
bool CreateFooter(std::string* output) override;
private:
// * A compressed log starts out empty (PRE_HEADER).
// * Once the header is produced, the stream is ACTIVE.
// * If it is ever detected that compressing the next input would exceed the
// budget, that input is NOT compressed, and the state becomes FULL, from
// which only writing the footer or discarding the file are allowed.
// * Writing the footer is allowed on an ACTIVE or FULL stream. Then, the
// stream is effectively closed.
// * Any error puts the stream into ERRORED. An errored stream can only
// be discarded.
enum class State { PRE_HEADER, ACTIVE, FULL, POST_FOOTER, ERRORED };
// Returns the budget left after reserving the GZIP overhead.
// Optionals without a value, both in the parameters as well as in the
// return value of the function, signal an unlimited amount.
static base::Optional<size_t> SizeAfterOverheadReservation(
base::Optional<size_t> max_size_bytes);
// Compresses |input| into |output|, while observing the budget (unless
// !budgeted). If |last|, also closes the stream.
Result CompressInternal(const std::string& input,
std::string* output,
bool budgeted,
bool last);
// Compresses the input data already in |stream_| into |output|.
bool Deflate(int flush, std::string* output);
State state_;
Budget budget_;
std::unique_ptr<CompressedSizeEstimator> compressed_size_estimator_;
z_stream stream_;
};
GzipLogCompressor::GzipLogCompressor(
base::Optional<size_t> max_size_bytes,
std::unique_ptr<CompressedSizeEstimator> compressed_size_estimator)
: state_(State::PRE_HEADER),
budget_(SizeAfterOverheadReservation(max_size_bytes)),
compressed_size_estimator_(std::move(compressed_size_estimator)) {
memset(&stream_, 0, sizeof(z_stream));
// Using (MAX_WBITS + 16) triggers the creation of a GZIP header.
const int result =
deflateInit2(&stream_, Z_DEFAULT_COMPRESSION, Z_DEFLATED, MAX_WBITS + 16,
kDefaultMemLevel, Z_DEFAULT_STRATEGY);
DCHECK_EQ(result, Z_OK);
}
GzipLogCompressor::~GzipLogCompressor() {
const int result = deflateEnd(&stream_);
// Z_DATA_ERROR reports that the stream was not properly terminated,
// but nevertheless correctly released. That happens when we don't
// write the footer.
DCHECK(result == Z_OK ||
(result == Z_DATA_ERROR && state_ != State::POST_FOOTER));
}
void GzipLogCompressor::CreateHeader(std::string* output) {
DCHECK(output);
DCHECK(output->empty());
DCHECK_EQ(state_, State::PRE_HEADER);
const Result result =
CompressInternal("", output, /*budgeted=*/false, /*last=*/false);
DCHECK_EQ(result, Result::OK);
DCHECK_EQ(output->size(), kGzipHeaderBytes);
state_ = State::ACTIVE;
}
LogCompressor::Result GzipLogCompressor::Compress(const std::string& input,
std::string* output) {
DCHECK_EQ(state_, State::ACTIVE);
if (input.empty()) {
return Result::OK;
}
const auto result =
CompressInternal(input, output, /*budgeted=*/true, /*last=*/false);
switch (result) {
case Result::OK:
return result;
case Result::DISALLOWED:
state_ = State::FULL;
return result;
case Result::ERROR_ENCOUNTERED:
state_ = State::ERRORED;
return result;
}
NOTREACHED();
return Result::ERROR_ENCOUNTERED; // Appease compiler.
}
bool GzipLogCompressor::CreateFooter(std::string* output) {
DCHECK(output);
DCHECK(output->empty());
DCHECK(state_ == State::ACTIVE || state_ == State::FULL);
const Result result =
CompressInternal("", output, /*budgeted=*/false, /*last=*/true);
if (result != Result::OK) { // !budgeted -> Result::DISALLOWED impossible.
DCHECK_EQ(result, Result::ERROR_ENCOUNTERED);
// An error message was logged by CompressInternal().
state_ = State::ERRORED;
return false;
}
if (output->length() != kGzipFooterBytes) {
LOG(ERROR) << "Incorrect footer size (" << output->length() << ").";
state_ = State::ERRORED;
return false;
}
state_ = State::POST_FOOTER;
return true;
}
base::Optional<size_t> GzipLogCompressor::SizeAfterOverheadReservation(
base::Optional<size_t> max_size_bytes) {
if (!max_size_bytes.has_value()) {
return base::Optional<size_t>();
} else {
DCHECK_GE(max_size_bytes.value(), kGzipHeaderBytes + kGzipFooterBytes);
return max_size_bytes.value() - (kGzipHeaderBytes + kGzipFooterBytes);
}
}
LogCompressor::Result GzipLogCompressor::CompressInternal(
const std::string& input,
std::string* output,
bool budgeted,
bool last) {
DCHECK(output);
DCHECK(output->empty());
DCHECK(state_ == State::PRE_HEADER || state_ == State::ACTIVE ||
(!budgeted && state_ == State::FULL));
// Avoid writing to |output| unless the return value is OK.
std::string temp_output;
if (budgeted) {
const size_t estimated_compressed_size =
compressed_size_estimator_->EstimateCompressedSize(input);
if (!budget_.ConsumeAllowed(estimated_compressed_size)) {
return Result::DISALLOWED;
}
}
if (last) {
DCHECK(input.empty());
stream_.next_in = nullptr;
} else {
stream_.next_in = reinterpret_cast<z_const Bytef*>(input.c_str());
}
DCHECK_LE(input.length(),
static_cast<size_t>(std::numeric_limits<uInt>::max()));
stream_.avail_in = static_cast<uInt>(input.length());
const bool result = Deflate(last ? Z_FINISH : Z_SYNC_FLUSH, &temp_output);
stream_.next_in = nullptr; // Avoid dangling pointers.
if (!result) {
// An error message was logged by Deflate().
return Result::ERROR_ENCOUNTERED;
}
if (budgeted) {
if (!budget_.ConsumeAllowed(temp_output.length())) {
LOG(WARNING) << "Compressed size was above estimate and unexpectedly "
"exceeded the budget.";
return Result::ERROR_ENCOUNTERED;
}
budget_.Consume(temp_output.length());
}
std::swap(*output, temp_output);
return Result::OK;
}
bool GzipLogCompressor::Deflate(int flush, std::string* output) {
DCHECK((flush != Z_FINISH && stream_.next_in != nullptr) ||
(flush == Z_FINISH && stream_.next_in == nullptr));
DCHECK(output->empty());
bool success = true; // Result of this method.
int z_result; // Result of the zlib function.
size_t total_compressed_size = 0;
do {
// Allocate some additional buffer.
constexpr uInt kCompressionBuffer = 4 * 1024;
output->resize(total_compressed_size + kCompressionBuffer);
// This iteration should write directly beyond previous iterations' last
// written byte.
stream_.next_out =
reinterpret_cast<uint8_t*>(&((*output)[total_compressed_size]));
stream_.avail_out = kCompressionBuffer;
z_result = deflate(&stream_, flush);
DCHECK_GE(kCompressionBuffer, stream_.avail_out);
const size_t compressed_size = kCompressionBuffer - stream_.avail_out;
if (flush != Z_FINISH) {
if (z_result != Z_OK) {
LOG(ERROR) << "Compression failed (" << z_result << ").";
success = false;
break;
}
} else { // flush == Z_FINISH
// End of the stream; we expect the footer to be exactly the size which
// we've set aside for it.
if (z_result != Z_STREAM_END || compressed_size != kGzipFooterBytes) {
LOG(ERROR) << "Compression failed (" << z_result << ", "
<< compressed_size << ").";
success = false;
break;
}
}
total_compressed_size += compressed_size;
} while (stream_.avail_out == 0 && z_result != Z_STREAM_END);
stream_.next_out = nullptr; // Avoid dangling pointers.
if (success) {
output->resize(total_compressed_size);
} else {
output->clear();
}
return success;
}
} // namespace
const size_t kGzipOverheadBytes = kGzipHeaderBytes + kGzipFooterBytes;
const base::FilePath::CharType kWebRtcEventLogUncompressedExtension[] =
FILE_PATH_LITERAL("log");
const base::FilePath::CharType kWebRtcEventLogGzippedExtension[] =
FILE_PATH_LITERAL("log.gz");
size_t BaseLogFileWriterFactory::MinFileSizeBytes() const {
// No overhead incurred; data written straight to the file without metadata.
......@@ -330,6 +710,82 @@ std::unique_ptr<LogFileWriter> BaseLogFileWriterFactory::Create(
return result;
}
std::unique_ptr<CompressedSizeEstimator>
DefaultGzippedSizeEstimator::Factory::Create() const {
return std::make_unique<DefaultGzippedSizeEstimator>();
}
size_t DefaultGzippedSizeEstimator::EstimateCompressedSize(
const std::string& input) const {
// This estimation is not tight. Since we expect to produce logs of
// several MBs, overshooting the estimation by one KB should be
// very safe and still relatively efficient.
constexpr size_t kOverheadOverUncompressedSizeBytes = 1000;
return input.length() + kOverheadOverUncompressedSizeBytes;
}
GzipLogCompressorFactory::GzipLogCompressorFactory(
std::unique_ptr<CompressedSizeEstimator::Factory> estimator_factory)
: estimator_factory_(std::move(estimator_factory)) {}
GzipLogCompressorFactory::~GzipLogCompressorFactory() = default;
size_t GzipLogCompressorFactory::MinSizeBytes() const {
return kGzipOverheadBytes;
}
std::unique_ptr<LogCompressor> GzipLogCompressorFactory::Create(
base::Optional<size_t> max_size_bytes) const {
if (max_size_bytes.has_value() && max_size_bytes.value() < MinSizeBytes()) {
LOG(WARNING) << "Max size (" << max_size_bytes.value()
<< ") below minimum size (" << MinSizeBytes() << ").";
return nullptr;
}
return std::make_unique<GzipLogCompressor>(max_size_bytes,
estimator_factory_->Create());
}
GzippedLogFileWriterFactory::GzippedLogFileWriterFactory(
std::unique_ptr<GzipLogCompressorFactory> gzip_compressor_factory)
: gzip_compressor_factory_(std::move(gzip_compressor_factory)) {}
GzippedLogFileWriterFactory::~GzippedLogFileWriterFactory() = default;
size_t GzippedLogFileWriterFactory::MinFileSizeBytes() const {
// Only the compression's own overhead is incurred.
return gzip_compressor_factory_->MinSizeBytes();
}
base::FilePath::StringPieceType GzippedLogFileWriterFactory::Extension() const {
return kWebRtcEventLogGzippedExtension;
}
std::unique_ptr<LogFileWriter> GzippedLogFileWriterFactory::Create(
const base::FilePath& path,
base::Optional<size_t> max_file_size_bytes) const {
if (max_file_size_bytes.has_value() &&
max_file_size_bytes.value() < MinFileSizeBytes()) {
LOG(WARNING) << "Size below allowed minimum.";
return nullptr;
}
auto gzip_compressor = gzip_compressor_factory_->Create(max_file_size_bytes);
if (!gzip_compressor) {
// The factory itself will have logged an error.
return nullptr;
}
auto result = std::make_unique<GzippedLogFileWriter>(
path, max_file_size_bytes, std::move(gzip_compressor));
if (!result->Init()) {
// Error logged by Init.
result.reset(); // Destructor deletes errored files.
}
return result;
}
BrowserContextId GetBrowserContextId(
const content::BrowserContext* browser_context) {
DCHECK_CURRENTLY_ON(content::BrowserThread::UI);
......
......@@ -43,16 +43,19 @@ extern const size_t kMaxActiveRemoteBoundWebRtcEventLogs;
// limit is applied per browser context.
extern const size_t kMaxPendingRemoteBoundWebRtcEventLogs;
// Overhead incurred by GZIP due to its header and footer.
extern const size_t kGzipOverheadBytes;
// Remote-bound log files' names will be of the format [prefix]_[log_id].[ext],
// where |prefix| is equal to kRemoteBoundWebRtcEventLogFileNamePrefix,
// |log_id| is composed of 32 random characters from '0'-'9' and 'A'-'F',
// and |ext| is the extension determined by the used LogCompressor::Factory,
// which will be either kWebRtcEventLogUncompressedExtension or
// kWebRtcEventLogGzippedExtension.
// TODO(crbug.com/775415): Add kWebRtcEventLogGzippedExtension.
extern const base::FilePath::CharType
kRemoteBoundWebRtcEventLogFileNamePrefix[];
extern const base::FilePath::CharType kWebRtcEventLogUncompressedExtension[];
extern const base::FilePath::CharType kWebRtcEventLogGzippedExtension[];
// Remote-bound event logs will not be uploaded if the time since their last
// modification (meaning the time when they were completed) exceeds this value.
......@@ -258,6 +261,148 @@ class BaseLogFileWriterFactory : public LogFileWriter::Factory {
base::Optional<size_t> max_file_size_bytes) const override;
};
// Interface for a class that provides compression of a stream, while attempting
// to observe a limit on the size.
//
// One should note that:
// * For compressors that use a footer, to guarantee proper decompression,
// the footer must be written to the file.
// * In such a case, usually, nothing can be omitted from the file, or the
// footer's CRC (if used) would be wrong.
// * Determining a string's size pre-compression, without performing the actual
// compression, is heuristic in nature.
//
// Therefore, compression might terminate (FULL) earlier than it
// must, or even in theory (which we attempt to avoid in practice) exceed the
// size allowed it, in which case the file will be discarded (ERROR).
class LogCompressor {
public:
// By subclassing this factory, concrete implementations of LogCompressor can
// be produced by unit tests, while keeping their definition in the .cc file.
// (Only the factory needs to be declared in the header.)
class Factory {
public:
virtual ~Factory() = default;
// The smallest size a log file of this type may assume.
virtual size_t MinSizeBytes() const = 0;
// Returns a LogCompressor if the parameters are valid and all
// initializations are successful; en empty unique_ptr otherwise.
// If !max_size_bytes.has_value(), an unlimited compressor is created.
virtual std::unique_ptr<LogCompressor> Create(
base::Optional<size_t> max_size_bytes) const = 0;
};
// Result of a call to Compress().
// * OK and ERROR_ENCOUNTERED are self-explanatory.
// * DISALLOWED means that, due to budget constraints, the input could
// not be compressed. The stream is still in a legal state, but only
// a call to CreateFooter() is now allowed.
enum class Result { OK, DISALLOWED, ERROR_ENCOUNTERED };
virtual ~LogCompressor() = default;
// Produces a compression header and writes it to |output|.
// The size does not count towards the max size limit.
// Guaranteed not to fail (nothing can realistically go wrong).
virtual void CreateHeader(std::string* output) = 0;
// Compresses |input| into |output|.
// * If compression succeeded, and the budget was observed, OK is returned.
// * If the compressor thinks the string, once compressed, will exceed the
// maximum size (when combined with previously compressed strings),
// compression will not be done, and DISALLOWED will be returned.
// This allows producing a valid footer without exceeding the size limit.
// * Unexpected errors in the underlying compressor (e.g. zlib, etc.),
// or unexpectedly getting a compressed string which exceeds the budget,
// will return ERROR_ENCOUNTERED.
// This function may not be called again if DISALLOWED or ERROR_ENCOUNTERED
// were ever returned before, or after CreateFooter() was called.
virtual Result Compress(const std::string& input, std::string* output) = 0;
// Produces a compression footer and writes it to |output|.
// The footer does not count towards the max size limit.
// May not be called more than once, or if Compress() returned ERROR.
virtual bool CreateFooter(std::string* output) = 0;
};
// Estimates the compressed size, without performing compression (except in
// unit tests, where performance is of lesser importance).
// This interface allows unit tests to simulate specific cases, such as
// over/under-estimation, and show that the code using the LogCompressor
// deals with them correctly. (E.g., if the estimation expects the compression
// to not go over-budget, but then it does.)
// The estimator is expected to be stateful. That is, the order of calls to
// EstimateCompressedSize() should correspond to the order of calls
// to Compress().
class CompressedSizeEstimator {
public:
class Factory {
public:
virtual ~Factory() = default;
virtual std::unique_ptr<CompressedSizeEstimator> Create() const = 0;
};
virtual ~CompressedSizeEstimator() = default;
virtual size_t EstimateCompressedSize(const std::string& input) const = 0;
};
// Provides a conservative estimation of the number of bytes required to
// compress a string using GZIP. This estimation is not expected to ever
// be overly optimistic, but the code using it should nevertheless be prepared
// to deal with that theoretical possibility.
class DefaultGzippedSizeEstimator : public CompressedSizeEstimator {
public:
class Factory : public CompressedSizeEstimator::Factory {
public:
~Factory() override = default;
std::unique_ptr<CompressedSizeEstimator> Create() const override;
};
~DefaultGzippedSizeEstimator() override = default;
size_t EstimateCompressedSize(const std::string& input) const override;
};
// Interface for producing LogCompressorGzip objects.
class GzipLogCompressorFactory : public LogCompressor::Factory {
public:
explicit GzipLogCompressorFactory(
std::unique_ptr<CompressedSizeEstimator::Factory> estimator_factory);
~GzipLogCompressorFactory() override;
size_t MinSizeBytes() const override;
std::unique_ptr<LogCompressor> Create(
base::Optional<size_t> max_size_bytes) const override;
private:
std::unique_ptr<CompressedSizeEstimator::Factory> estimator_factory_;
};
// Produces LogFileWriter instances that perform compression using GZIP.
class GzippedLogFileWriterFactory : public LogFileWriter::Factory {
public:
explicit GzippedLogFileWriterFactory(
std::unique_ptr<GzipLogCompressorFactory> gzip_compressor_factory);
~GzippedLogFileWriterFactory() override;
size_t MinFileSizeBytes() const override;
base::FilePath::StringPieceType Extension() const override;
std::unique_ptr<LogFileWriter> Create(
const base::FilePath& path,
base::Optional<size_t> max_file_size_bytes) const override;
private:
std::unique_ptr<GzipLogCompressorFactory> gzip_compressor_factory_;
};
// Translate a BrowserContext into an ID. This lets us associate PeerConnections
// with BrowserContexts, while making sure that we never call the
// BrowserContext's methods outside of the UI thread (because we can't call them
......
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "chrome/browser/media/webrtc/webrtc_event_log_manager_common.h"
#include <memory>
#include <numeric>
#include <vector>
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/files/scoped_temp_dir.h"
#include "base/optional.h"
#include "base/rand_util.h"
#include "base/test/scoped_task_environment.h"
#include "build/build_config.h"
#include "chrome/browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/zlib/google/compression_utils.h"
namespace {
constexpr LogCompressor::Result OK = LogCompressor::Result::OK;
constexpr LogCompressor::Result DISALLOWED = LogCompressor::Result::DISALLOWED;
constexpr LogCompressor::Result ERROR_ENCOUNTERED =
LogCompressor::Result::ERROR_ENCOUNTERED;
} // namespace
// Tests for GzipLogCompressor.
// Note that these tests may not use GzippedSize(), or they would be assuming
// what they set out to prove. (Subsequent tests may use it, though.)
class GzipLogCompressorTest : public ::testing::Test {
public:
~GzipLogCompressorTest() override = default;
void Init(
std::unique_ptr<CompressedSizeEstimator::Factory> estimator_factory) {
DCHECK(!compressor_factory_);
DCHECK(estimator_factory);
compressor_factory_ = std::make_unique<GzipLogCompressorFactory>(
std::move(estimator_factory));
}
std::string Decompress(const std::string& input) {
std::string output;
EXPECT_TRUE(compression::GzipUncompress(input, &output));
return output;
}
std::unique_ptr<GzipLogCompressorFactory> compressor_factory_;
};
TEST_F(GzipLogCompressorTest,
GzipLogCompressorFactoryCreatesCompressorIfMinimalSizeOrAbove) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
const size_t min_size = compressor_factory_->MinSizeBytes();
auto compressor = compressor_factory_->Create(min_size);
EXPECT_TRUE(compressor);
}
TEST_F(GzipLogCompressorTest,
GzipLogCompressorFactoryDoesNotCreateCompressorIfBelowMinimalSize) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
const size_t min_size = compressor_factory_->MinSizeBytes();
ASSERT_GE(min_size, 1u);
auto compressor = compressor_factory_->Create(min_size - 1);
EXPECT_FALSE(compressor);
}
TEST_F(GzipLogCompressorTest, EmptyStreamReasonableMaxSize) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
auto compressor = compressor_factory_->Create(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
std::string footer;
ASSERT_TRUE(compressor->CreateFooter(&footer));
const std::string simulated_file = header + footer;
EXPECT_EQ(Decompress(simulated_file), "");
}
TEST_F(GzipLogCompressorTest, EmptyStreamMinimalSize) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
const size_t min_size = compressor_factory_->MinSizeBytes();
auto compressor = compressor_factory_->Create(min_size);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
std::string footer;
ASSERT_TRUE(compressor->CreateFooter(&footer));
const std::string simulated_file = header + footer;
EXPECT_EQ(Decompress(simulated_file), "");
}
TEST_F(GzipLogCompressorTest, SingleCallToCompress) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
auto compressor = compressor_factory_->Create(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
const std::string input = "Some random text.";
std::string log;
ASSERT_EQ(compressor->Compress(input, &log), OK);
std::string footer;
ASSERT_TRUE(compressor->CreateFooter(&footer));
const std::string simulated_file = header + log + footer;
EXPECT_EQ(Decompress(simulated_file), input);
}
TEST_F(GzipLogCompressorTest, MultipleCallsToCompress) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
auto compressor = compressor_factory_->Create(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
const std::vector<std::string> inputs = {
"Some random text.",
"This text is also random. I give you my word for it. 100% random.",
"nejnnc pqmnx0981 mnl<D@ikjed90~~,z."};
std::vector<std::string> logs(inputs.size());
for (size_t i = 0; i < inputs.size(); i++) {
ASSERT_EQ(compressor->Compress(inputs[i], &logs[i]), OK);
}
std::string footer;
ASSERT_TRUE(compressor->CreateFooter(&footer));
const auto input = std::accumulate(begin(inputs), end(inputs), std::string());
const auto log = std::accumulate(begin(logs), end(logs), std::string());
const std::string simulated_file = header + log + footer;
EXPECT_EQ(Decompress(simulated_file), input);
}
TEST_F(GzipLogCompressorTest, UnlimitedBudgetSanity) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
auto compressor = compressor_factory_->Create(base::Optional<size_t>());
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
const std::string input = "Some random text.";
std::string log;
ASSERT_EQ(compressor->Compress(input, &log), OK);
std::string footer;
ASSERT_TRUE(compressor->CreateFooter(&footer));
const std::string simulated_file = header + log + footer;
EXPECT_EQ(Decompress(simulated_file), input);
}
// Test once with a big input, to provide coverage over inputs that could
// exceed the size of some local buffers in the UUT.
TEST_F(GzipLogCompressorTest, CompressionBigInput) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
auto compressor = compressor_factory_->Create(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
constexpr size_t kRealisticSizeBytes = 1000 * 1000;
const std::string input = base::RandBytesAsString(kRealisticSizeBytes);
std::string log;
ASSERT_EQ(compressor->Compress(input, &log), OK);
std::string footer;
ASSERT_TRUE(compressor->CreateFooter(&footer));
const std::string simulated_file = header + log + footer;
EXPECT_EQ(Decompress(simulated_file), input);
}
TEST_F(GzipLogCompressorTest, BudgetExceededByFirstCompressYieldsEmptyFile) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
const std::string input = "This won't fit.";
auto compressor = compressor_factory_->Create(GzippedSize(input) - 1);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
// Focal point #1 - Compress() returns DISALLOWED.
std::string log;
EXPECT_EQ(compressor->Compress(input, &log), DISALLOWED);
// Focal point #2 - CreateFooter() still succeeds;
std::string footer;
EXPECT_TRUE(compressor->CreateFooter(&footer));
// Focal point #3 - the resulting log is parsable, and contains only those
// logs for which Compress() was successful.
// Note that |log| is not supposed to be written to the file, because
// Compress() has disallowed it.
const std::string simulated_file = header + footer;
EXPECT_EQ(Decompress(simulated_file), "");
}
TEST_F(GzipLogCompressorTest,
BudgetExceededByNonFirstCompressYieldsPartialFile) {
Init(std::make_unique<PerfectGzipEstimator::Factory>());
const std::string short_input = "short";
const std::string long_input = "A somewhat longer input string. @$%^&*()!!2";
// Allocate enough budget that |short_input| would be produced, and not yet
// exhaust the budget, but |long_input| won't fit.
auto compressor = compressor_factory_->Create(GzippedSize(short_input) + 1);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
std::string short_log;
ASSERT_EQ(compressor->Compress(short_input, &short_log), OK);
// Focal point #1 - Compress() returns DISALLOWED.
std::string long_log;
EXPECT_EQ(compressor->Compress(long_input, &long_log), DISALLOWED);
EXPECT_TRUE(long_log.empty());
// Focal point #2 - CreateFooter() still succeeds;
std::string footer;
EXPECT_TRUE(compressor->CreateFooter(&footer));
// Focal point #3 - the resulting log is parsable, and contains only those
// logs for which Compress() was successful.
// Note that |long_log| is not supposed to be written to the file, because
// Compress() has disallowed it.
const std::string simulated_file = header + short_log + footer;
EXPECT_EQ(Decompress(simulated_file), short_input);
}
TEST_F(GzipLogCompressorTest,
ExceedingBudgetDueToOverlyOptimisticEstimationYieldsError) {
// Use an estimator that will always be overly optimistic.
Init(std::make_unique<NullEstimator::Factory>());
// Set a budget that will easily be exceeded.
auto compressor = compressor_factory_->Create(kGzipOverheadBytes + 5);
ASSERT_TRUE(compressor);
std::string header;
compressor->CreateHeader(&header);
// Prepare to compress an input that is guaranteed to exceed the budget.
const std::string input = "A string that would not fit in five bytes.";
// The estimation allowed the compression, but then the compressed output
// ended up being over-budget.
std::string compressed;
EXPECT_EQ(compressor->Compress(input, &compressed), ERROR_ENCOUNTERED);
EXPECT_TRUE(compressed.empty());
}
// Tests relevant to all LogFileWriter subclasses.
class LogFileWriterTest
: public ::testing::Test,
public ::testing::WithParamInterface<WebRtcEventLogCompression> {
public:
LogFileWriterTest() { EXPECT_TRUE(temp_dir_.CreateUniqueTempDir()); }
~LogFileWriterTest() override {}
void Init(WebRtcEventLogCompression compression) {
DCHECK(!compression_.has_value()) << "Must only be called once.";
compression_ = compression;
log_file_writer_factory_ = CreateLogFileWriterFactory(compression);
path_ = temp_dir_.GetPath()
.Append(FILE_PATH_LITERAL("arbitrary_filename"))
.AddExtension(log_file_writer_factory_->Extension());
}
std::unique_ptr<LogFileWriter> CreateWriter(base::Optional<size_t> max_size) {
return log_file_writer_factory_->Create(path_, max_size);
}
void ExpectFileContents(const base::FilePath& file_path,
const std::string& expected_contents) {
DCHECK(compression_.has_value()) << "Must call Init().";
std::string file_contents;
ASSERT_TRUE(base::ReadFileToString(file_path, &file_contents));
switch (compression_.value()) {
case WebRtcEventLogCompression::NONE: {
EXPECT_EQ(file_contents, expected_contents);
break;
}
case WebRtcEventLogCompression::GZIP_PERFECT_ESTIMATION:
case WebRtcEventLogCompression::GZIP_NULL_ESTIMATION: {
std::string uncompressed;
ASSERT_TRUE(compression::GzipUncompress(file_contents, &uncompressed));
EXPECT_EQ(uncompressed, expected_contents);
break;
}
default: { NOTREACHED(); }
}
}
base::test::ScopedTaskEnvironment scoped_task_environment_;
base::Optional<WebRtcEventLogCompression> compression_; // Set in Init().
base::ScopedTempDir temp_dir_;
base::FilePath path_;
std::unique_ptr<LogFileWriter::Factory> log_file_writer_factory_;
};
TEST_P(LogFileWriterTest, FactoryCreatesLogFileWriter) {
Init(GetParam());
EXPECT_TRUE(CreateWriter(log_file_writer_factory_->MinFileSizeBytes()));
}
#if defined(OS_POSIX)
TEST_P(LogFileWriterTest, FactoryReturnsEmptyUniquePtrIfCantCreateFile) {
Init(GetParam());
RemoveWritePermissions(temp_dir_.GetPath());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
EXPECT_FALSE(writer);
}
#endif // defined(OS_POSIX)
TEST_P(LogFileWriterTest, CloseSucceedsWhenNoErrorsOccurred) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
EXPECT_TRUE(writer->Close());
}
// Other tests check check the case of compression where the estimation is
// close to the file's capacity, reaches or exceeds it.
TEST_P(LogFileWriterTest, CallToWriteSuccedsWhenCapacityFarOff) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
const std::string log = "log";
EXPECT_TRUE(writer->Write(log));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, log);
}
TEST_P(LogFileWriterTest, CallToWriteWithEmptyStringSucceeds) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
const std::string log = "";
EXPECT_TRUE(writer->Write(log));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, log);
}
TEST_P(LogFileWriterTest, UnlimitedBudgetSanity) {
Init(GetParam());
auto writer = CreateWriter(base::Optional<size_t>());
ASSERT_TRUE(writer);
const std::string log = "log";
EXPECT_TRUE(writer->Write(log));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, log);
}
TEST_P(LogFileWriterTest, DeleteRemovesUnclosedFile) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
writer->Delete();
EXPECT_FALSE(base::PathExists(path_));
}
TEST_P(LogFileWriterTest, DeleteRemovesClosedFile) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
EXPECT_TRUE(writer->Close());
writer->Delete();
EXPECT_FALSE(base::PathExists(path_));
}
#if !defined(OS_WIN) // Deleting the open file does not work on Windows.
TEST_P(LogFileWriterTest, WriteDoesNotCrashIfFileRemovedExternally) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
ASSERT_TRUE(base::DeleteFile(path_, /*recursive=*/false));
ASSERT_FALSE(base::PathExists(path_)); // Sanity on the test itself.
// It's up to the OS whether this will succeed or fail, but it must not crash.
writer->Write("log");
}
TEST_P(LogFileWriterTest, CloseDoesNotCrashIfFileRemovedExternally) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
ASSERT_TRUE(base::DeleteFile(path_, /*recursive=*/false));
ASSERT_FALSE(base::PathExists(path_)); // Sanity on the test itself.
// It's up to the OS whether this will succeed or fail, but it must not crash.
writer->Close();
}
TEST_P(LogFileWriterTest, DeleteDoesNotCrashIfFileRemovedExternally) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
ASSERT_TRUE(base::DeleteFile(path_, /*recursive=*/false));
ASSERT_FALSE(base::PathExists(path_)); // Sanity on the test itself.
// It's up to the OS whether this will succeed or fail, but it must not crash.
writer->Delete();
}
#endif // !defined(OS_WIN)
TEST_P(LogFileWriterTest, PathReturnsTheCorrectPath) {
Init(GetParam());
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
ASSERT_EQ(writer->path(), path_);
}
INSTANTIATE_TEST_CASE_P(
Compression,
LogFileWriterTest,
::testing::Values(WebRtcEventLogCompression::NONE,
WebRtcEventLogCompression::GZIP_PERFECT_ESTIMATION));
// Tests for UncompressedLogFileWriterTest only.
class UncompressedLogFileWriterTest : public LogFileWriterTest {
public:
~UncompressedLogFileWriterTest() override = default;
};
TEST_F(UncompressedLogFileWriterTest,
MaxSizeReachedReturnsFalseWhenMaxNotReached) {
Init(WebRtcEventLogCompression::NONE);
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
const std::string log = "log";
ASSERT_TRUE(writer->Write(log));
EXPECT_FALSE(writer->MaxSizeReached());
}
TEST_F(UncompressedLogFileWriterTest, MaxSizeReachedReturnsTrueWhenMaxReached) {
Init(WebRtcEventLogCompression::NONE);
const std::string log = "log";
auto writer = CreateWriter(log.size());
ASSERT_TRUE(writer);
ASSERT_TRUE(writer->Write(log)); // (CallToWriteSuccedsWhenCapacityReached)
EXPECT_TRUE(writer->MaxSizeReached());
}
TEST_F(UncompressedLogFileWriterTest, CallToWriteSuccedsWhenCapacityReached) {
Init(WebRtcEventLogCompression::NONE);
const std::string log = "log";
auto writer = CreateWriter(log.size());
ASSERT_TRUE(writer);
EXPECT_TRUE(writer->Write(log));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, log);
}
TEST_F(UncompressedLogFileWriterTest, CallToWriteFailsWhenCapacityExceeded) {
Init(WebRtcEventLogCompression::NONE);
const std::string log = "log";
auto writer = CreateWriter(log.size() - 1);
ASSERT_TRUE(writer);
EXPECT_FALSE(writer->Write(log));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, "");
}
TEST_F(UncompressedLogFileWriterTest, WriteCompleteMessagesOnly) {
Init(WebRtcEventLogCompression::NONE);
const std::string log1 = "01234";
const std::string log2 = "56789";
auto writer = CreateWriter(log1.size() + log2.size() - 1);
ASSERT_TRUE(writer);
EXPECT_TRUE(writer->Write(log1));
EXPECT_FALSE(writer->Write(log2));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, log1);
}
// Tests for GzippedLogFileWriterTest only.
class GzippedLogFileWriterTest : public LogFileWriterTest {
public:
~GzippedLogFileWriterTest() override = default;
};
TEST_F(GzippedLogFileWriterTest, FactoryDeletesFileIfMaxSizeBelowMin) {
Init(WebRtcEventLogCompression::GZIP_NULL_ESTIMATION);
const size_t min_size = log_file_writer_factory_->MinFileSizeBytes();
ASSERT_GE(min_size, 1u);
auto writer = CreateWriter(min_size - 1);
ASSERT_FALSE(writer);
EXPECT_FALSE(base::PathExists(path_));
}
TEST_F(GzippedLogFileWriterTest, MaxSizeReachedReturnsFalseWhenMaxNotReached) {
Init(WebRtcEventLogCompression::GZIP_NULL_ESTIMATION);
auto writer = CreateWriter(kMaxRemoteLogFileSizeBytes);
ASSERT_TRUE(writer);
const std::string log = "log";
ASSERT_TRUE(writer->Write(log));
EXPECT_FALSE(writer->MaxSizeReached());
}
TEST_F(GzippedLogFileWriterTest, MaxSizeReachedReturnsTrueWhenMaxReached) {
// By using a 0 estimation, we allow the compressor to keep going to
// the point of budget saturation.
Init(WebRtcEventLogCompression::GZIP_NULL_ESTIMATION);
const std::string log = "log";
auto writer = CreateWriter(GzippedSize(log));
ASSERT_TRUE(writer);
ASSERT_TRUE(writer->Write(log)); // (CallToWriteSuccedsWhenCapacityReached)
EXPECT_TRUE(writer->MaxSizeReached());
}
TEST_F(GzippedLogFileWriterTest, CallToWriteSuccedsWhenCapacityReached) {
Init(WebRtcEventLogCompression::GZIP_PERFECT_ESTIMATION);
const std::string log = "log";
auto writer = CreateWriter(GzippedSize(log));
ASSERT_TRUE(writer);
EXPECT_TRUE(writer->Write(log));
ASSERT_TRUE(writer->Close());
ExpectFileContents(path_, log);
}
// Also tests the scenario WriteCompleteMessagesOnly.
TEST_F(GzippedLogFileWriterTest,
CallToWriteFailsWhenCapacityWouldBeExceededButEstimationPreventedWrite) {
Init(WebRtcEventLogCompression::GZIP_PERFECT_ESTIMATION);
const std::string log1 = "abcde";
const std::string log2 = "fghij";
const std::vector<std::string> logs = {log1, log2};
// Find out the size necessary for compressing log1 and log2 in two calls.
const size_t compressed_len = GzippedSize(logs); // Vector version.
auto writer = CreateWriter(compressed_len - 1);
ASSERT_TRUE(writer);
ASSERT_TRUE(writer->Write(log1));
EXPECT_FALSE(writer->Write(log2));
// The second write was succesfully prevented; no error should have occurred,
// and it should be possible to produce a meaningful gzipped log file.
EXPECT_TRUE(writer->Close());
ExpectFileContents(path_, log1); // Only the in-budget part was written.
}
// This tests the case when the estimation fails to warn us of a pending
// over-budget write, which leaves us unable to produce a valid compression
// footer for the truncated file. This forces us to discard the file.
TEST_F(GzippedLogFileWriterTest,
CallToWriteFailsWhenCapacityExceededDespiteEstimationAllowingIt) {
// By using a 0 estimation, we allow the compressor to keep going to
// the point of budget saturation.
Init(WebRtcEventLogCompression::GZIP_NULL_ESTIMATION);
const std::string log = "log";
auto writer = CreateWriter(GzippedSize(log) - 1);
ASSERT_TRUE(writer);
EXPECT_FALSE(writer->Write(log));
EXPECT_FALSE(writer->Close());
EXPECT_FALSE(base::PathExists(path_)); // Errored files deleted by Close().
}
......@@ -537,15 +537,19 @@ void WebRtcRemoteEventLogManager::AddPendingLogs(
const base::FilePath& remote_bound_logs_dir) {
DCHECK(task_runner_->RunsTasksInCurrentSequence());
base::FilePath::StringType pattern =
base::FilePath(FILE_PATH_LITERAL("*"))
.AddExtension(log_file_writer_factory_->Extension())
.value();
base::FileEnumerator enumerator(remote_bound_logs_dir,
/*recursive=*/false,
base::FileEnumerator::FILES, pattern);
base::FileEnumerator::FILES);
for (auto path = enumerator.Next(); !path.empty(); path = enumerator.Next()) {
const base::FileEnumerator::FileInfo info = enumerator.GetInfo();
const base::FilePath::StringType extension = info.GetName().Extension();
const auto separator =
base::FilePath::StringType(1, base::FilePath::kExtensionSeparator);
if (extension != separator + kWebRtcEventLogUncompressedExtension &&
extension != separator + kWebRtcEventLogGzippedExtension) {
continue;
}
const auto last_modified = enumerator.GetInfo().GetLastModifiedTime();
auto it = pending_logs_.emplace(browser_context_id, path, last_modified);
DCHECK(it.second); // No pre-existing entry.
......
......@@ -340,7 +340,6 @@ class WebRtcRemoteEventLogManager final
std::map<PeerConnectionKey, const std::string> active_peer_connections_;
// Creates LogFileWriter instances (compressed/uncompressed, etc.).
// TODO(crbug.com/775415): Add support for compressed version using GZIP.
std::unique_ptr<LogFileWriter::Factory> log_file_writer_factory_;
// Remote-bound logs which we're currently in the process of writing to disk.
......
......@@ -21,10 +21,10 @@
#include "base/files/file_path.h"
#include "base/files/file_util.h"
#include "base/files/scoped_temp_dir.h"
#include "base/macros.h"
#include "base/memory/scoped_refptr.h"
#include "base/optional.h"
#include "base/run_loop.h"
#include "base/stl_util.h"
#include "base/strings/string_number_conversions.h"
#include "base/strings/string_util.h"
#include "base/synchronization/waitable_event.h"
......@@ -48,6 +48,7 @@
#include "net/url_request/url_request_test_util.h"
#include "testing/gmock/include/gmock/gmock.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/zlib/google/compression_utils.h"
// TODO(crbug.com/775415): Add unit tests for incognito mode.
// TODO(crbug.com/775415): Migrate to being based on Profiles rather than on
......@@ -255,8 +256,8 @@ class WebRtcEventLogManagerTestBase : public ::testing::Test {
event_log_manager_->SetRemoteLogFileWriterFactoryForTesting(
std::move(factory));
} else {
// TODO(crbug.com/775415): Add GZIP support and make it on by default.
remote_log_extension_ = kWebRtcEventLogUncompressedExtension;
// kWebRtcRemoteEventLogGzipped is turned on by default.
remote_log_extension_ = kWebRtcEventLogGzippedExtension;
}
}
......@@ -573,9 +574,16 @@ class WebRtcEventLogManagerTestBase : public ::testing::Test {
std::string file_contents;
ASSERT_TRUE(base::ReadFileToString(file_path, &file_contents));
// TODO(crbug.com/775415): Support compression.
DCHECK(remote_log_extension_ == kWebRtcEventLogUncompressedExtension);
EXPECT_EQ(file_contents, expected_event_log);
if (remote_log_extension_ == kWebRtcEventLogUncompressedExtension) {
EXPECT_EQ(file_contents, expected_event_log);
} else if (remote_log_extension_ == kWebRtcEventLogGzippedExtension) {
std::string uncompressed_log;
ASSERT_TRUE(
compression::GzipUncompress(file_contents, &uncompressed_log));
EXPECT_EQ(uncompressed_log, expected_event_log);
} else {
NOTREACHED();
}
}
// When the peer connection's ID is not the focus of the test, this allows
......@@ -699,7 +707,7 @@ class WebRtcEventLogManagerTest : public WebRtcEventLogManagerTestBase,
~WebRtcEventLogManagerTest() override = default;
void SetUp() override {
CreateWebRtcEventLogManager(Compression::NONE);
CreateWebRtcEventLogManager(Compression::GZIP_PERFECT_ESTIMATION);
auto tracker = std::make_unique<content::MockNetworkConnectionTracker>(
true, network::mojom::ConnectionType::CONNECTION_ETHERNET);
......@@ -918,7 +926,35 @@ class WebRtcEventLogManagerTestUploadDelay
static const size_t kIntentionallyExcessiveDelayMs = 1000 * 1000 * 1000;
};
// For testing compression issues.
class WebRtcEventLogManagerTestCompression
: public WebRtcEventLogManagerTestBase {
public:
WebRtcEventLogManagerTestCompression() {
scoped_feature_list_.InitAndEnableFeature(features::kWebRtcRemoteEventLog);
scoped_command_line_.GetProcessCommandLine()->AppendSwitchASCII(
::switches::kWebRtcRemoteEventLogUploadDelayMs, "0");
}
~WebRtcEventLogManagerTestCompression() override = default;
void SetUp() override {
// Defer until Init(), which will allow the test body more control.
}
void Init(base::Optional<WebRtcEventLogCompression> remote_compression =
base::Optional<WebRtcEventLogCompression>()) {
CreateWebRtcEventLogManager(remote_compression);
auto tracker = std::make_unique<content::MockNetworkConnectionTracker>(
true, network::mojom::ConnectionType::CONNECTION_ETHERNET);
WebRtcEventLogManagerTestBase::SetUp(std::move(tracker));
}
};
namespace {
class PeerConnectionTrackerProxyForTesting
: public WebRtcEventLogManager::PeerConnectionTrackerProxy {
public:
......@@ -1749,6 +1785,21 @@ TEST_F(WebRtcEventLogManagerTest,
kMaxRemoteLogFileSizeBytes));
}
TEST_F(WebRtcEventLogManagerTest,
StartRemoteLoggingReturnsFalseIfFileSizeToSmall) {
const size_t min_size =
CreateLogFileWriterFactory(Compression::GZIP_NULL_ESTIMATION)
->MinFileSizeBytes();
const auto key = GetPeerConnectionKey(rph_.get(), kLid);
const std::string id = "id"; // For explicitness' sake.
ASSERT_TRUE(PeerConnectionAdded(key.render_process_id, key.lid, id));
std::string error_message;
EXPECT_FALSE(StartRemoteLogging(key.render_process_id, id, min_size - 1,
nullptr, &error_message));
EXPECT_EQ(error_message, kStartRemoteLoggingFailureMaxSizeTooSmall);
}
TEST_F(WebRtcEventLogManagerTest,
StartRemoteLoggingReturnsFalseIfExcessivelyLargeFileSize) {
const auto key = GetPeerConnectionKey(rph_.get(), kLid);
......@@ -1865,7 +1916,7 @@ TEST_F(WebRtcEventLogManagerTest, RemoteLogFileCreatedInCorrectDirectory) {
}
// All log files must be created in their own context's directory.
for (size_t i = 0; i < arraysize(browser_contexts); ++i) {
for (size_t i = 0; i < base::size(browser_contexts); ++i) {
ASSERT_TRUE(file_paths[i]);
EXPECT_TRUE(browser_contexts[i]->GetPath().IsParent(*file_paths[i]));
}
......@@ -1983,18 +2034,18 @@ TEST_F(WebRtcEventLogManagerTest, MultipleWritesToSameRemoteBoundLogfile) {
std::accumulate(std::begin(logs), std::end(logs), std::string()));
}
TEST_F(WebRtcEventLogManagerTest, RemoteLogFileSizeLimitNotExceeded) {
TEST_F(WebRtcEventLogManagerTest,
RemoteLogFileSizeLimitNotExceededSingleWrite) {
const auto key = GetPeerConnectionKey(rph_.get(), kLid);
base::Optional<base::FilePath> file_path;
ON_CALL(remote_observer_, OnRemoteLogStarted(key, _))
.WillByDefault(Invoke(SaveFilePathTo(&file_path)));
const std::string log = "tpyo";
const size_t file_size_limit_bytes = log.length() / 2;
ASSERT_TRUE(PeerConnectionAdded(key.render_process_id, key.lid));
ASSERT_TRUE(StartRemoteLogging(key.render_process_id, GetUniqueId(key),
file_size_limit_bytes));
GzippedSize(log) - 1));
// Failure is reported, because not everything could be written. The file
// will also be closed.
......@@ -2002,13 +2053,40 @@ TEST_F(WebRtcEventLogManagerTest, RemoteLogFileSizeLimitNotExceeded) {
ASSERT_EQ(OnWebRtcEventLogWrite(key.render_process_id, key.lid, log),
std::make_pair(false, false));
// Additional calls to Write() have no effect.
ASSERT_EQ(OnWebRtcEventLogWrite(key.render_process_id, key.lid, "ignored"),
std::make_pair(false, false));
// Make sure the file would be closed, so that we could safely read it.
ASSERT_TRUE(PeerConnectionRemoved(key.render_process_id, key.lid));
// No partial writes occurred.
ExpectRemoteFileContents(*file_path, "");
}
TEST_F(WebRtcEventLogManagerTest,
RemoteLogFileSizeLimitNotExceededMultipleWrites) {
const auto key = GetPeerConnectionKey(rph_.get(), kLid);
base::Optional<base::FilePath> file_path;
ON_CALL(remote_observer_, OnRemoteLogStarted(key, _))
.WillByDefault(Invoke(SaveFilePathTo(&file_path)));
const std::string log1 = "abcabc";
const std::string log2 = "defghijklmnopqrstuvwxyz";
ASSERT_TRUE(PeerConnectionAdded(key.render_process_id, key.lid));
ASSERT_TRUE(StartRemoteLogging(key.render_process_id, GetUniqueId(key),
1 + GzippedSize(log1)));
// First write works.
ASSERT_EQ(OnWebRtcEventLogWrite(key.render_process_id, key.lid, log1),
std::make_pair(false, true));
// On the second write, failure is reported, because not everything could be
// written. The file will also be closed.
EXPECT_CALL(remote_observer_, OnRemoteLogStopped(key)).Times(1);
ASSERT_EQ(OnWebRtcEventLogWrite(key.render_process_id, key.lid, log2),
std::make_pair(false, false));
ExpectRemoteFileContents(*file_path, log1);
}
TEST_F(WebRtcEventLogManagerTest,
LogMultipleActiveRemoteLogsSameBrowserContext) {
const std::vector<PeerConnectionKey> keys = {
......@@ -2091,9 +2169,9 @@ TEST_F(WebRtcEventLogManagerTest,
TEST_F(WebRtcEventLogManagerTest, DifferentRemoteLogsMayHaveDifferentMaximums) {
const std::string logs[2] = {"abra", "cadabra"};
std::vector<base::Optional<base::FilePath>> file_paths(arraysize(logs));
std::vector<base::Optional<base::FilePath>> file_paths(base::size(logs));
std::vector<PeerConnectionKey> keys;
for (size_t i = 0; i < arraysize(logs); ++i) {
for (size_t i = 0; i < base::size(logs); ++i) {
keys.push_back(GetPeerConnectionKey(rph_.get(), i));
ON_CALL(remote_observer_, OnRemoteLogStarted(keys[i], _))
.WillByDefault(Invoke(SaveFilePathTo(&file_paths[i])));
......@@ -2102,7 +2180,7 @@ TEST_F(WebRtcEventLogManagerTest, DifferentRemoteLogsMayHaveDifferentMaximums) {
for (size_t i = 0; i < keys.size(); ++i) {
ASSERT_TRUE(PeerConnectionAdded(keys[i].render_process_id, keys[i].lid));
ASSERT_TRUE(StartRemoteLogging(keys[i].render_process_id,
GetUniqueId(keys[i]), logs[i].length()));
GetUniqueId(keys[i]), GzippedSize(logs[i])));
}
for (size_t i = 0; i < keys.size(); ++i) {
......@@ -2127,7 +2205,7 @@ TEST_F(WebRtcEventLogManagerTest, RemoteLogFileClosedWhenCapacityReached) {
ASSERT_TRUE(PeerConnectionAdded(key.render_process_id, key.lid));
ASSERT_TRUE(StartRemoteLogging(key.render_process_id, GetUniqueId(key),
log.length()));
GzippedSize(log)));
ASSERT_TRUE(file_path);
EXPECT_CALL(remote_observer_, OnRemoteLogStopped(key)).Times(1);
......@@ -2229,7 +2307,7 @@ TEST_F(WebRtcEventLogManagerTest,
ASSERT_TRUE(PeerConnectionAdded(key.render_process_id, key.lid));
EXPECT_CALL(remote_observer_, OnRemoteLogStarted(key, _)).Times(1);
ASSERT_TRUE(StartRemoteLogging(key.render_process_id, GetUniqueId(key),
log.length()));
GzippedSize(log)));
}
// By writing to one of the logs until it reaches capacity, we fill it,
......@@ -2368,6 +2446,53 @@ TEST_F(WebRtcEventLogManagerTest,
WaitForPendingTasks(&run_loop);
}
// It is possible for remote-bound logs to be compressed or uncompressed.
// We show that logs from a previous session are captured even if they are
// different, with regards to compression, compared to last time.
TEST_F(WebRtcEventLogManagerTest,
LogsCapturedPreviouslyMadePendingEvenIfDifferentExtensionUsed) {
// Unload the profile, but remember where it stores its files.
const base::FilePath browser_context_path = browser_context_->GetPath();
const base::FilePath remote_logs_dir = RemoteBoundLogsDir(browser_context_);
UnloadProfiles();
// Seed the remote logs' directory with log files, simulating the
// creation of logs in a previous session.
std::list<WebRtcLogFileInfo> expected_files;
ASSERT_TRUE(CreateDirectory(remote_logs_dir));
base::FilePath::StringPieceType extensions[] = {
kWebRtcEventLogUncompressedExtension, kWebRtcEventLogGzippedExtension};
ASSERT_LE(base::size(extensions), kMaxPendingRemoteBoundWebRtcEventLogs)
<< "Lacking test coverage.";
for (size_t i = 0, ext = 0; i < kMaxPendingRemoteBoundWebRtcEventLogs; ++i) {
const auto& extension = extensions[ext];
ext = (ext + 1) % base::size(extensions);
const base::FilePath file_path =
remote_logs_dir.Append(IntToStringType(i)).AddExtension(extension);
constexpr int file_flags = base::File::FLAG_CREATE |
base::File::FLAG_WRITE |
base::File::FLAG_EXCLUSIVE_WRITE;
base::File file(file_path, file_flags);
ASSERT_TRUE(file.IsValid() && file.created());
expected_files.emplace_back(browser_context_id_, file_path,
GetLastModificationTime(file_path));
}
// This factory enforces the expectation that the files will be uploaded,
// all of them, only them, and in the order expected.
base::RunLoop run_loop;
SetWebRtcEventLogUploaderFactoryForTesting(
std::make_unique<FileListExpectingWebRtcEventLogUploader::Factory>(
&expected_files, true, &run_loop));
LoadProfiles();
ASSERT_EQ(browser_context_->GetPath(), browser_context_path);
WaitForPendingTasks(&run_loop);
}
TEST_P(WebRtcEventLogManagerTest,
WhenPeerConnectionRemovedFinishedRemoteLogUploadedAndFileDeleted) {
// |upload_result| show that the files are deleted independent of the
......@@ -3613,6 +3738,42 @@ TEST_F(WebRtcEventLogManagerTestUploadDelay,
WaitForPendingTasks(&run_loop);
}
TEST_F(WebRtcEventLogManagerTestCompression,
ErroredFilesDueToBadEstimationDeletedRatherThanUploaded) {
Init(Compression::GZIP_NULL_ESTIMATION);
const std::string log = "It's better than bad; it's good.";
const auto key = GetPeerConnectionKey(rph_.get(), kLid);
base::Optional<base::FilePath> log_file;
ON_CALL(remote_observer_, OnRemoteLogStarted(key, _))
.WillByDefault(Invoke(SaveFilePathTo(&log_file)));
ASSERT_TRUE(PeerConnectionAdded(key.render_process_id, key.lid));
ASSERT_TRUE(StartRemoteLogging(key.render_process_id, GetUniqueId(key),
GzippedSize(log) - 1));
ASSERT_TRUE(log_file);
std::list<WebRtcLogFileInfo> empty_list;
base::RunLoop run_loop;
SetWebRtcEventLogUploaderFactoryForTesting(
std::make_unique<FileListExpectingWebRtcEventLogUploader::Factory>(
&empty_list, true, &run_loop));
// Writing fails because the budget is exceeded.
EXPECT_EQ(OnWebRtcEventLogWrite(key.render_process_id, key.lid, log),
std::make_pair(false, false));
// The file was deleted due to the error we've instigated (by using an
// intentionally over-optimistic estimation).
EXPECT_FALSE(base::PathExists(*log_file));
// If the file is incorrectly still eligible for an upload, this will trigger
// the upload (which will be a test failure).
ASSERT_TRUE(PeerConnectionRemoved(key.render_process_id, key.lid));
WaitForPendingTasks(&run_loop);
}
#else // defined(OS_ANDROID)
class WebRtcEventLogManagerTestOnMobileDevices
......
......@@ -14,8 +14,15 @@ std::unique_ptr<LogFileWriter::Factory> CreateLogFileWriterFactory(
switch (compression) {
case WebRtcEventLogCompression::NONE:
return std::make_unique<BaseLogFileWriterFactory>();
case WebRtcEventLogCompression::GZIP_NULL_ESTIMATION:
return std::make_unique<GzippedLogFileWriterFactory>(
std::make_unique<GzipLogCompressorFactory>(
std::make_unique<NullEstimator::Factory>()));
case WebRtcEventLogCompression::GZIP_PERFECT_ESTIMATION:
return std::make_unique<GzippedLogFileWriterFactory>(
std::make_unique<GzipLogCompressorFactory>(
std::make_unique<PerfectGzipEstimator::Factory>()));
}
NOTREACHED();
return nullptr; // Appease compiler.
}
......@@ -31,3 +38,57 @@ void RemoveWritePermissions(const base::FilePath& path) {
ASSERT_TRUE(base::SetPosixFilePermissions(path, permissions));
}
#endif // defined(OS_POSIX)
std::unique_ptr<CompressedSizeEstimator> NullEstimator::Factory::Create()
const {
return std::make_unique<NullEstimator>();
}
size_t NullEstimator::EstimateCompressedSize(const std::string& input) const {
return 0;
}
std::unique_ptr<CompressedSizeEstimator> PerfectGzipEstimator::Factory::Create()
const {
return std::make_unique<PerfectGzipEstimator>();
}
PerfectGzipEstimator::PerfectGzipEstimator() {
// This factory will produce an optimistic compressor that will always
// think it can compress additional inputs, which will therefore allow
// us to find out what the real compressed size it, since compression
// will never be suppressed.
GzipLogCompressorFactory factory(std::make_unique<NullEstimator::Factory>());
compressor_ = factory.Create(base::Optional<size_t>());
DCHECK(compressor_);
std::string ignored;
compressor_->CreateHeader(&ignored);
}
PerfectGzipEstimator::~PerfectGzipEstimator() = default;
size_t PerfectGzipEstimator::EstimateCompressedSize(
const std::string& input) const {
std::string output;
EXPECT_EQ(compressor_->Compress(input, &output), LogCompressor::Result::OK);
return output.length();
}
size_t GzippedSize(const std::string& uncompressed) {
PerfectGzipEstimator perfect_estimator;
return kGzipOverheadBytes +
perfect_estimator.EstimateCompressedSize(uncompressed);
}
size_t GzippedSize(const std::vector<std::string>& uncompressed) {
PerfectGzipEstimator perfect_estimator;
size_t result = kGzipOverheadBytes;
for (const std::string& str : uncompressed) {
result += perfect_estimator.EstimateCompressedSize(str);
}
return result;
}
......@@ -6,6 +6,7 @@
#define CHROME_BROWSER_MEDIA_WEBRTC_WEBRTC_EVENT_LOG_MANAGER_UNITTEST_HELPERS_H_
#include <memory>
#include <string>
#include "base/files/file_path.h"
#include "build/build_config.h"
......@@ -13,8 +14,9 @@
// Which type of compression, if any, LogFileWriterTest should use.
enum class WebRtcEventLogCompression {
NONE
// TODO(crbug.com/775415): Add support for GZIP.
NONE,
GZIP_NULL_ESTIMATION,
GZIP_PERFECT_ESTIMATION
};
// Produce a LogFileWriter::Factory object.
......@@ -25,4 +27,52 @@ std::unique_ptr<LogFileWriter::Factory> CreateLogFileWriterFactory(
void RemoveWritePermissions(const base::FilePath& path);
#endif // defined(OS_POSIX)
// Always estimates strings to be compressed to zero bytes.
class NullEstimator : public CompressedSizeEstimator {
public:
class Factory : public CompressedSizeEstimator::Factory {
public:
~Factory() override = default;
std::unique_ptr<CompressedSizeEstimator> Create() const override;
};
~NullEstimator() override = default;
size_t EstimateCompressedSize(const std::string& input) const override;
};
// Provides a perfect estimation of the compressed size by cheating - performing
// actual compression, then reporting the resulting size.
// This class is stateful; the number, nature and order of calls to
// EstimateCompressedSize() is important.
class PerfectGzipEstimator : public CompressedSizeEstimator {
public:
class Factory : public CompressedSizeEstimator::Factory {
public:
~Factory() override = default;
std::unique_ptr<CompressedSizeEstimator> Create() const override;
};
PerfectGzipEstimator();
~PerfectGzipEstimator() override;
size_t EstimateCompressedSize(const std::string& input) const override;
private:
// This compressor allows EstimateCompressedSize to return an exact estimate.
// EstimateCompressedSize is normally const, but here we fake it, so we set
// it as mutable.
mutable std::unique_ptr<LogCompressor> compressor_;
};
// Check the gzipped size of |uncompressed|, including header and footer,
// assuming it were gzipped on its own.
size_t GzippedSize(const std::string& uncompressed);
// Same as other version, but with elements compressed in sequence.
size_t GzippedSize(const std::vector<std::string>& uncompressed);
#endif // CHROME_BROWSER_MEDIA_WEBRTC_WEBRTC_EVENT_LOG_MANAGER_UNITTEST_HELPERS_H_
......@@ -655,6 +655,9 @@ const base::Feature kMachineLearningService{"MachineLearningService",
// Please note that a Chrome policy must also be set, for this to have effect.
extern const base::Feature kWebRtcRemoteEventLog{
"WebRtcRemoteEventLog", base::FEATURE_DISABLED_BY_DEFAULT};
// Compress remote-bound WebRTC event logs (if used; see kWebRtcRemoteEventLog).
extern const base::Feature kWebRtcRemoteEventLogGzipped{
"WebRtcRemoteEventLogGzipped", base::FEATURE_ENABLED_BY_DEFAULT};
#endif
#if defined(OS_WIN)
......
......@@ -355,6 +355,7 @@ extern const base::Feature kMachineLearningService;
#if !defined(OS_ANDROID)
extern const base::Feature kWebRtcRemoteEventLog;
extern const base::Feature kWebRtcRemoteEventLogGzipped;
#endif
#if defined(OS_WIN)
......
......@@ -2989,6 +2989,7 @@ test("unit_tests") {
"../browser/media/router/discovery/discovery_network_monitor_metric_observer_unittest.cc",
"../browser/media/router/discovery/discovery_network_monitor_unittest.cc",
"../browser/media/webrtc/tab_desktop_media_list_unittest.cc",
"../browser/media/webrtc/webrtc_event_log_manager_common_unittest.cc",
"../browser/media/webrtc/webrtc_event_log_manager_unittest.cc",
"../browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.cc",
"../browser/media/webrtc/webrtc_event_log_manager_unittest_helpers.h",
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment