Add MerkleIntegritySourceStream

This adds MerkleIntegritySourceStream class which decodes and validates
the "mi-sha256" content encoding [1]. This class will be used by
WebPackageLoader to verify integrity of message body.

[1] https://tools.ietf.org/html/draft-thomson-http-mice-02

Bug: 803774
Change-Id: I62353f594e30e90fa982a68374389267b39c21fe
Reviewed-on: https://chromium-review.googlesource.com/892844
Commit-Queue: Kunihiko Sakamoto <ksakamoto@chromium.org>
Reviewed-by: Tsuyoshi Horo <horo@chromium.org>
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Kouhei Ueno <kouhei@chromium.org>
Cr-Commit-Position: refs/heads/master@{#533203}

content/browser/BUILD.gn

@@ -979,6 +979,8 @@ jumbo_source_set("browser") {
     "loader/loader_delegate.h",
     "loader/loader_io_thread_notifier.cc",
     "loader/loader_io_thread_notifier.h",
+    "loader/merkle_integrity_source_stream.cc",
+    "loader/merkle_integrity_source_stream.h",
     "loader/mime_sniffing_resource_handler.cc",
     "loader/mime_sniffing_resource_handler.h",
     "loader/mojo_async_resource_handler.cc",

content/browser/loader/merkle_integrity_source_stream.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/loader/merkle_integrity_source_stream.h"
+
+#include "base/base64url.h"
+#include "base/big_endian.h"
+#include "crypto/sha2.h"
+#include "net/base/io_buffer.h"
+
+namespace content {
+
+namespace {
+
+// Limit the record size to 5MiB to prevent browser OOM.
+constexpr uint64_t kMaxRecordSize = 5 * 1024 * 1024;
+
+constexpr char kMiSha256Header[] = "mi-sha256=";
+constexpr int kMiSha256HeaderLength = sizeof(kMiSha256Header) - 1;
+
+}  // namespace
+
+MerkleIntegritySourceStream::MerkleIntegritySourceStream(
+    const std::string& mi_header_value,
+    std::unique_ptr<SourceStream> upstream)
+    // TODO(ksakamoto): Use appropriate SourceType.
+    : net::FilterSourceStream(SourceStream::TYPE_NONE, std::move(upstream)),
+      record_size_(0),
+      failed_(false) {
+  // TODO(ksakamoto): Support quoted parameter value.
+  if (mi_header_value.substr(0, kMiSha256HeaderLength) != kMiSha256Header ||
+      !base::Base64UrlDecode(mi_header_value.substr(kMiSha256HeaderLength),
+                             base::Base64UrlDecodePolicy::DISALLOW_PADDING,
+                             &next_proof_) ||
+      next_proof_.size() != crypto::kSHA256Length) {
+    failed_ = true;
+  }
+}
+
+int MerkleIntegritySourceStream::FilterData(net::IOBuffer* output_buffer,
+                                            int output_buffer_size,
+                                            net::IOBuffer* input_buffer,
+                                            int input_buffer_size,
+                                            int* consumed_bytes,
+                                            bool upstream_eof_reached) {
+  if (failed_)
+    return net::ERR_CONTENT_DECODING_FAILED;
+
+  // TODO(ksakamoto): Avoid unnecessary buffer copying.
+  input_.append(input_buffer->data(), input_buffer_size);
+  *consumed_bytes = input_buffer_size;
+
+  if (!ProcessInput(upstream_eof_reached)) {
+    failed_ = true;
+    return net::ERR_CONTENT_DECODING_FAILED;
+  }
+
+  int bytes_out =
+      std::min(output_.size(), static_cast<size_t>(output_buffer_size));
+  output_.copy(output_buffer->data(), bytes_out);
+  output_.erase(0, bytes_out);
+  return bytes_out;
+}
+
+bool MerkleIntegritySourceStream::ProcessInput(bool upstream_eof_reached) {
+  // TODO(ksakamoto): Use shift iterator or StringPiece instead of substr/erase.
+
+  // Read the record size (the first 8 octets of the stream).
+  if (!record_size_) {
+    if (input_.size() < 8)
+      return !upstream_eof_reached;
+
+    base::ReadBigEndian(input_.data(), &record_size_);
+    input_.erase(0, 8);
+    if (record_size_ == 0)
+      return false;
+    if (record_size_ > kMaxRecordSize) {
+      DVLOG(1)
+          << "Rejecting MI content encoding because record size is too big: "
+          << record_size_;
+      return false;
+    }
+  }
+
+  // Process records other than the last.
+  while (input_.size() >= record_size_ + crypto::kSHA256Length) {
+    std::string chunk = input_.substr(0, record_size_ + crypto::kSHA256Length);
+    input_.erase(0, record_size_ + crypto::kSHA256Length);
+    chunk.push_back('\x01');
+    std::string hash = crypto::SHA256HashString(chunk);
+    if (next_proof_ != hash)
+      return false;
+    output_.append(chunk.substr(0, record_size_));
+    next_proof_ = chunk.substr(record_size_, crypto::kSHA256Length);
+  }
+
+  // Process the last record.
+  if (upstream_eof_reached && !next_proof_.empty()) {
+    if (input_.size() > record_size_)
+      return false;
+
+    input_.push_back('\0');
+    std::string hash = crypto::SHA256HashString(input_);
+    if (next_proof_ != hash)
+      return false;
+
+    output_.append(input_.substr(0, input_.size() - 1));
+    input_.clear();
+    next_proof_.clear();
+  }
+  return true;
+}
+
+std::string MerkleIntegritySourceStream::GetTypeAsString() const {
+  return "MI-256";
+}
+
+}  // namespace content

content/browser/loader/merkle_integrity_source_stream.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_BROWSER_LOADER_MERKLE_INTEGRITY_SOURCE_STREAM_H_
+#define CONTENT_BROWSER_LOADER_MERKLE_INTEGRITY_SOURCE_STREAM_H_
+
+#include <stdint.h>
+#include <string>
+
+#include "base/macros.h"
+#include "content/common/content_export.h"
+#include "net/filter/filter_source_stream.h"
+
+namespace content {
+
+// MerkleIntegritySourceStream decodes and validates content encoded with the
+// "mi-sha256" content encoding
+// (https://tools.ietf.org/html/draft-thomson-http-mice-02).
+// TODO(ksakamoto): This class should eventually live in src/net/filter/.
+class CONTENT_EXPORT MerkleIntegritySourceStream
+    : public net::FilterSourceStream {
+ public:
+  MerkleIntegritySourceStream(const std::string& mi_header_value,
+                              std::unique_ptr<SourceStream> upstream);
+
+  // net::FilterSourceStream
+  int FilterData(net::IOBuffer* output_buffer,
+                 int output_buffer_size,
+                 net::IOBuffer* input_buffer,
+                 int input_buffer_size,
+                 int* consumed_bytes,
+                 bool upstream_eof_reached) override;
+  std::string GetTypeAsString() const override;
+
+ private:
+  bool ProcessInput(bool upstream_eof_reached);
+
+  std::string input_;
+  std::string output_;
+  // SHA-256 hash for the next record, or empty if validation is completed.
+  std::string next_proof_;
+  uint64_t record_size_;
+  bool failed_;
+
+  DISALLOW_COPY_AND_ASSIGN(MerkleIntegritySourceStream);
+};
+
+}  // namespace content
+
+#endif  // CONTENT_BROWSER_LOADER_MERKLE_INTEGRITY_SOURCE_STREAM_H_

content/browser/loader/merkle_integrity_source_stream_unittest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/loader/merkle_integrity_source_stream.h"
+#include "base/base64url.h"
+#include "net/base/io_buffer.h"
+#include "net/base/test_completion_callback.h"
+#include "net/filter/mock_source_stream.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace content {
+
+namespace {
+
+const int kBigBufferSize = 4096;
+const int kSmallBufferSize = 1;
+
+const char kMIEmptyBody[] =
+    "mi-sha256=bjQLnP-zepicpUTmu3gKLHiQHT-zNzh2hRGjBhevoB0";
+const char kMISingleRecord[] =
+    "mi-sha256=dcRDgR2GM35DluAV13PzgnG6-pvQwPywfFvAu1UeFrs";
+const char kMIMultipleRecords[] =
+    "mi-sha256=IVa9shfs0nyKEhHqtB3WVNANJ2Njm5KjQLjRtnbkYJ4";
+
+enum class ReadResultType {
+  // Each call to AddReadResult is a separate read from the lower layer
+  // SourceStream.
+  EVERYTHING_AT_ONCE,
+  // Whenever AddReadResult is called, each byte is actually a separate read
+  // result.
+  ONE_BYTE_AT_A_TIME,
+};
+
+struct MerkleIntegrityTestParam {
+  MerkleIntegrityTestParam(int buf_size,
+                           net::MockSourceStream::Mode read_mode,
+                           ReadResultType read_result_type)
+      : buffer_size(buf_size),
+        mode(read_mode),
+        read_result_type(read_result_type) {}
+
+  const int buffer_size;
+  const net::MockSourceStream::Mode mode;
+  const ReadResultType read_result_type;
+};
+
+}  // namespace
+
+class MerkleIntegritySourceStreamTest
+    : public ::testing::TestWithParam<MerkleIntegrityTestParam> {
+ protected:
+  MerkleIntegritySourceStreamTest()
+      : output_buffer_size_(GetParam().buffer_size) {}
+
+  void Init(const std::string& mi_header_value) {
+    output_buffer_ = new net::IOBuffer(output_buffer_size_);
+    std::unique_ptr<net::MockSourceStream> source(new net::MockSourceStream());
+    if (GetParam().read_result_type == ReadResultType::ONE_BYTE_AT_A_TIME)
+      source->set_read_one_byte_at_a_time(true);
+    source_ = source.get();
+    stream_ = std::make_unique<MerkleIntegritySourceStream>(mi_header_value,
+                                                            std::move(source));
+  }
+
+  // If MockSourceStream::Mode is ASYNC, completes reads from |mock_stream|
+  // until there's no pending read, and then returns |callback|'s result, once
+  // it's invoked. If Mode is not ASYNC, does nothing and returns
+  // |previous_result|.
+  int CompleteReadsIfAsync(int previous_result,
+                           net::TestCompletionCallback* callback,
+                           net::MockSourceStream* mock_stream) {
+    if (GetParam().mode == net::MockSourceStream::ASYNC) {
+      EXPECT_EQ(net::ERR_IO_PENDING, previous_result);
+      while (mock_stream->awaiting_completion())
+        mock_stream->CompleteNextRead();
+      return callback->WaitForResult();
+    }
+    return previous_result;
+  }
+
+  net::IOBuffer* output_buffer() { return output_buffer_.get(); }
+  char* output_data() { return output_buffer_->data(); }
+  size_t output_buffer_size() { return output_buffer_size_; }
+
+  net::MockSourceStream* source() { return source_; }
+  MerkleIntegritySourceStream* stream() { return stream_.get(); }
+
+  // Reads from |stream_| until an error occurs or the EOF is reached.
+  // When an error occurs, returns the net error code. When an EOF is reached,
+  // returns the number of bytes read and appends data read to |output|.
+  int ReadStream(std::string* output) {
+    int bytes_read = 0;
+    while (true) {
+      net::TestCompletionCallback callback;
+      int rv = stream_->Read(output_buffer(), output_buffer_size(),
+                             callback.callback());
+      if (rv == net::ERR_IO_PENDING)
+        rv = CompleteReadsIfAsync(rv, &callback, source());
+      if (rv == net::OK)
+        break;
+      if (rv < net::OK)
+        return rv;
+      EXPECT_GT(rv, net::OK);
+      bytes_read += rv;
+      output->append(output_data(), rv);
+    }
+    return bytes_read;
+  }
+
+  std::string Base64Decode(const char* hash) {
+    std::string out;
+    EXPECT_TRUE(base::Base64UrlDecode(
+        hash, base::Base64UrlDecodePolicy::DISALLOW_PADDING, &out));
+    EXPECT_EQ(32u, out.size());
+    return out;
+  }
+
+ private:
+  scoped_refptr<net::IOBuffer> output_buffer_;
+  int output_buffer_size_;
+
+  net::MockSourceStream* source_;
+  std::unique_ptr<MerkleIntegritySourceStream> stream_;
+};
+
+INSTANTIATE_TEST_CASE_P(
+    MerkleIntegritySourceStreamTests,
+    MerkleIntegritySourceStreamTest,
+    ::testing::Values(
+        MerkleIntegrityTestParam(kBigBufferSize,
+                                 net::MockSourceStream::SYNC,
+                                 ReadResultType::EVERYTHING_AT_ONCE),
+        MerkleIntegrityTestParam(kSmallBufferSize,
+                                 net::MockSourceStream::SYNC,
+                                 ReadResultType::EVERYTHING_AT_ONCE),
+        MerkleIntegrityTestParam(kBigBufferSize,
+                                 net::MockSourceStream::ASYNC,
+                                 ReadResultType::EVERYTHING_AT_ONCE),
+        MerkleIntegrityTestParam(kSmallBufferSize,
+                                 net::MockSourceStream::ASYNC,
+                                 ReadResultType::EVERYTHING_AT_ONCE),
+        MerkleIntegrityTestParam(kBigBufferSize,
+                                 net::MockSourceStream::SYNC,
+                                 ReadResultType::ONE_BYTE_AT_A_TIME),
+        MerkleIntegrityTestParam(kSmallBufferSize,
+                                 net::MockSourceStream::SYNC,
+                                 ReadResultType::ONE_BYTE_AT_A_TIME),
+        MerkleIntegrityTestParam(kBigBufferSize,
+                                 net::MockSourceStream::ASYNC,
+                                 ReadResultType::ONE_BYTE_AT_A_TIME),
+        MerkleIntegrityTestParam(kSmallBufferSize,
+                                 net::MockSourceStream::ASYNC,
+                                 ReadResultType::ONE_BYTE_AT_A_TIME)));
+
+TEST_P(MerkleIntegritySourceStreamTest, EmptyStream) {
+  Init(kMIEmptyBody);
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int result = ReadStream(&actual_output);
+  EXPECT_EQ(net::ERR_CONTENT_DECODING_FAILED, result);
+}
+
+TEST_P(MerkleIntegritySourceStreamTest, MalformedMIHeader) {
+  Init("invalid-MI-header-value");
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int result = ReadStream(&actual_output);
+  EXPECT_EQ(net::ERR_CONTENT_DECODING_FAILED, result);
+}
+
+TEST_P(MerkleIntegritySourceStreamTest, RecordSizeOnly) {
+  Init(kMIEmptyBody);
+  const uint8_t record_size[] = {0, 0, 0, 0, 0, 0, 0, 10};
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int result = ReadStream(&actual_output);
+  EXPECT_EQ(net::OK, result);
+}
+
+TEST_P(MerkleIntegritySourceStreamTest, RecordSizeOnlyWrongHash) {
+  Init(kMISingleRecord);
+  const uint8_t record_size[] = {0, 0, 0, 0, 0, 0, 0, 10};
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int result = ReadStream(&actual_output);
+  EXPECT_EQ(net::ERR_CONTENT_DECODING_FAILED, result);
+}
+
+TEST_P(MerkleIntegritySourceStreamTest, RecordSizeTooBig) {
+  Init(kMIEmptyBody);
+  const uint8_t record_size[] = {0xff, 0xff, 0xff, 0xff,
+                                 0xff, 0xff, 0xff, 0xff};
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  std::string actual_output;
+  int result = ReadStream(&actual_output);
+  EXPECT_EQ(net::ERR_CONTENT_DECODING_FAILED, result);
+}
+
+// https://tools.ietf.org/html/draft-thomson-http-mice-02#section-4.1
+TEST_P(MerkleIntegritySourceStreamTest, SingleRecord) {
+  Init(kMISingleRecord);
+  const uint8_t record_size[] = {0, 0, 0, 0, 0, 0, 0, 0x29};
+  const std::string message("When I grow up, I want to be a watermelon");
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  source()->AddReadResult(message.data(), message.size(), net::OK,
+                          GetParam().mode);
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int rv = ReadStream(&actual_output);
+  EXPECT_EQ(static_cast<int>(message.size()), rv);
+  EXPECT_EQ(message, actual_output);
+}
+
+TEST_P(MerkleIntegritySourceStreamTest, SingleRecordWrongHash) {
+  Init(kMIEmptyBody);
+  const uint8_t record_size[] = {0, 0, 0, 0, 0, 0, 0, 0x29};
+  const std::string message("When I grow up, I want to be a watermelon");
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  source()->AddReadResult(message.data(), message.size(), net::OK,
+                          GetParam().mode);
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int rv = ReadStream(&actual_output);
+  EXPECT_EQ(net::ERR_CONTENT_DECODING_FAILED, rv);
+  EXPECT_EQ(0u, actual_output.size());
+}
+
+// https://tools.ietf.org/html/draft-thomson-http-mice-02#section-4.2
+TEST_P(MerkleIntegritySourceStreamTest, MultipleRecords) {
+  Init(kMIMultipleRecords);
+  const uint8_t record_size[] = {0, 0, 0, 0, 0, 0, 0, 16};
+  const std::string message("When I grow up, I want to be a watermelon");
+
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  source()->AddReadResult(message.data(), 16, net::OK, GetParam().mode);
+  std::string hash1 =
+      Base64Decode("OElbplJlPK-Rv6JNK6p5_515IaoPoZo-2elWL7OQ60A");
+  source()->AddReadResult(hash1.data(), 32, net::OK, GetParam().mode);
+  source()->AddReadResult(message.data() + 16, 16, net::OK, GetParam().mode);
+  std::string hash2 =
+      Base64Decode("iPMpmgExHPrbEX3_RvwP4d16fWlK4l--p75PUu_KyN0");
+  source()->AddReadResult(hash2.data(), 32, net::OK, GetParam().mode);
+  source()->AddReadResult(message.data() + 32, message.size() - 32, net::OK,
+                          GetParam().mode);
+
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+  std::string actual_output;
+  int rv = ReadStream(&actual_output);
+  EXPECT_EQ(static_cast<int>(message.size()), rv);
+  EXPECT_EQ(message, actual_output);
+}
+
+TEST_P(MerkleIntegritySourceStreamTest, MultipleRecordsWrongLastRecordHash) {
+  Init(kMIMultipleRecords);
+  const uint8_t record_size[] = {0, 0, 0, 0, 0, 0, 0, 16};
+  const std::string message("When I grow up, I want to be a watermelon!");
+
+  source()->AddReadResult(reinterpret_cast<const char*>(record_size),
+                          sizeof(record_size), net::OK, GetParam().mode);
+  source()->AddReadResult(message.data(), 16, net::OK, GetParam().mode);
+  std::string hash1 =
+      Base64Decode("OElbplJlPK-Rv6JNK6p5_515IaoPoZo-2elWL7OQ60A");
+  source()->AddReadResult(hash1.data(), 32, net::OK, GetParam().mode);
+  source()->AddReadResult(message.data() + 16, 16, net::OK, GetParam().mode);
+  std::string hash2 =
+      Base64Decode("iPMpmgExHPrbEX3_RvwP4d16fWlK4l--p75PUu_KyN0");
+  source()->AddReadResult(hash2.data(), 32, net::OK, GetParam().mode);
+  source()->AddReadResult(message.data() + 32, message.size() - 32, net::OK,
+                          GetParam().mode);
+  source()->AddReadResult(nullptr, 0, net::OK, GetParam().mode);
+
+  std::string actual_output;
+  int rv = ReadStream(&actual_output);
+  EXPECT_EQ(net::ERR_CONTENT_DECODING_FAILED, rv);
+  EXPECT_EQ(message.substr(0, 32), actual_output);
+}
+
+}  // namespace content

content/test/BUILD.gn

@@ -1303,6 +1303,7 @@ test("content_unittests") {
     "../browser/loader/cross_site_document_resource_handler_unittest.cc",
     "../browser/loader/detachable_resource_handler_unittest.cc",
     "../browser/loader/intercepting_resource_handler_unittest.cc",
+    "../browser/loader/merkle_integrity_source_stream_unittest.cc",
     "../browser/loader/mime_sniffing_resource_handler_unittest.cc",
     "../browser/loader/mock_resource_loader.cc",
     "../browser/loader/mock_resource_loader.h",