Commit 20e2cd90 authored by Dan Zhang's avatar Dan Zhang Committed by Commit Bot

Synchronise with internal hpack huffman encoder implementation.

Merge http2 part of internal change: 210537018

R=rch@chromium.org

Change-Id: Ia31231c598b3d7d8cb79e7b5ebd6f035d3d7b1fa
Reviewed-on: https://chromium-review.googlesource.com/1227838Reviewed-by: default avatarBence Béky <bnc@chromium.org>
Reviewed-by: default avatarRyan Hamilton <rch@chromium.org>
Commit-Queue: Dan Zhang <danzh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#592045}
parent 36de8d57
......@@ -1195,6 +1195,10 @@ component("net") {
"third_party/http2/hpack/http2_hpack_constants.h",
"third_party/http2/hpack/huffman/hpack_huffman_decoder.cc",
"third_party/http2/hpack/huffman/hpack_huffman_decoder.h",
"third_party/http2/hpack/huffman/hpack_huffman_encoder.cc",
"third_party/http2/hpack/huffman/hpack_huffman_encoder.h",
"third_party/http2/hpack/huffman/huffman_spec_tables.cc",
"third_party/http2/hpack/huffman/huffman_spec_tables.h",
"third_party/http2/hpack/varint/hpack_varint_decoder.cc",
"third_party/http2/hpack/varint/hpack_varint_decoder.h",
"third_party/http2/hpack/varint/hpack_varint_encoder.cc",
......@@ -4919,6 +4923,8 @@ test("net_unittests") {
"third_party/http2/hpack/hpack_string_test.cc",
"third_party/http2/hpack/http2_hpack_constants_test.cc",
"third_party/http2/hpack/huffman/hpack_huffman_decoder_test.cc",
"third_party/http2/hpack/huffman/hpack_huffman_encoder_test.cc",
"third_party/http2/hpack/huffman/hpack_huffman_transcoder_test.cc",
"third_party/http2/hpack/tools/hpack_block_builder.cc",
"third_party/http2/hpack/tools/hpack_block_builder.h",
"third_party/http2/hpack/tools/hpack_block_builder_test.cc",
......@@ -4932,6 +4938,9 @@ test("net_unittests") {
"third_party/http2/http2_structures_test_util.cc",
"third_party/http2/http2_structures_test_util.h",
"third_party/http2/platform/api/http2_string_utils_test.cc",
"third_party/http2/platform/api/random_util_helper.h",
"third_party/http2/platform/impl/random_util_helper_impl.cc",
"third_party/http2/platform/impl/random_util_helper_impl.h",
"third_party/http2/test_tools/frame_parts.cc",
"third_party/http2/test_tools/frame_parts.h",
"third_party/http2/test_tools/frame_parts_collector.cc",
......
#include "net/third_party/http2/hpack/huffman/hpack_huffman_encoder.h"
#include "base/logging.h"
#include "net/third_party/http2/hpack/huffman/huffman_spec_tables.h"
// TODO(jamessynge): Remove use of binary literals, that is a C++ 14 feature.
namespace http2 {
size_t ExactHuffmanSize(Http2StringPiece plain) {
size_t bits = 0;
for (const uint8_t c : plain) {
bits += HuffmanSpecTables::kCodeLengths[c];
}
return (bits + 7) / 8;
}
size_t BoundedHuffmanSize(Http2StringPiece plain) {
// TODO(jamessynge): Determine whether we should set the min size for Huffman
// encoding much higher (i.e. if less than N, then the savings isn't worth
// the cost of encoding and decoding). Of course, we need to decide on a
// value function, which might be throughput on a full load test, or a
// microbenchmark of the time to encode and then decode a HEADERS frame,
// possibly with the cost of crypto included (i.e. crypto is going to have
// a fairly constant per-byte cost, so reducing the number of bytes in-transit
// reduces the number that must be encrypted and later decrypted).
if (plain.size() < 3) {
// Huffman encoded string can't be smaller than the plain size for very
// short strings.
return plain.size();
}
// TODO(jamessynge): Measure whether this can be done more efficiently with
// nested loops (e.g. make exact measurement of 8 bytes, then check if min
// remaining is too long).
// Compute the number of bits in an encoding that is shorter than the plain
// string (i.e. the number of bits in a string 1 byte shorter than plain),
// and use this as the limit of the size of the encoding.
const size_t limit_bits = (plain.size() - 1) * 8;
// The shortest code length in the Huffman table of the HPACK spec has 5 bits
// (e.g. for 0, 1, a and e).
const size_t min_code_length = 5;
// We can therefore say that all plain text bytes whose code length we've not
// yet looked up will take at least 5 bits.
size_t min_bits_remaining = plain.size() * min_code_length;
size_t bits = 0;
for (const uint8_t c : plain) {
bits += HuffmanSpecTables::kCodeLengths[c];
min_bits_remaining -= min_code_length;
// If our minimum estimate of the total number of bits won't yield an
// encoding shorter the plain text, let's bail.
const size_t minimum_bits_total = bits + min_bits_remaining;
if (minimum_bits_total > limit_bits) {
bits += min_bits_remaining;
break;
}
}
return (bits + 7) / 8;
}
void HuffmanEncode(Http2StringPiece plain, Http2String* huffman) {
DCHECK(huffman != nullptr);
huffman->clear(); // Note that this doesn't release memory.
uint64_t bit_buffer = 0; // High-bit is next bit to output. Not clear if that
// is more performant than having the low-bit be the
// last to be output.
size_t bits_unused = 64; // Number of bits available for the next code.
for (uint8_t c : plain) {
size_t code_length = HuffmanSpecTables::kCodeLengths[c];
if (bits_unused < code_length) {
// There isn't enough room in bit_buffer for the code of c.
// Flush until bits_unused > 56 (i.e. 64 - 8).
do {
char h = static_cast<char>(bit_buffer >> 56);
bit_buffer <<= 8;
bits_unused += 8;
// Perhaps would be more efficient if we populated an array of chars,
// so we don't have to call push_back each time. Reconsider if used
// for production.
huffman->push_back(h);
} while (bits_unused <= 56);
}
uint64_t code = HuffmanSpecTables::kRightCodes[c];
size_t shift_by = bits_unused - code_length;
bit_buffer |= (code << shift_by);
bits_unused -= code_length;
}
// bit_buffer contains (64-bits_unused) bits that still need to be flushed.
// Output whole bytes until we don't have any whole bytes left.
size_t bits_used = 64 - bits_unused;
while (bits_used >= 8) {
char h = static_cast<char>(bit_buffer >> 56);
bit_buffer <<= 8;
bits_used -= 8;
huffman->push_back(h);
}
if (bits_used > 0) {
// We have less than a byte left to output. The spec calls for padding out
// the final byte with the leading bits of the EOS symbol (30 1-bits).
constexpr uint64_t leading_eos_bits = 0b11111111;
bit_buffer |= (leading_eos_bits << (56 - bits_used));
char h = static_cast<char>(bit_buffer >> 56);
huffman->push_back(h);
}
}
} // namespace http2
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef NET_THIRD_PARTY_HTTP2_HPACK_HUFFMAN_HPACK_HUFFMAN_ENCODER_H_
#define NET_THIRD_PARTY_HTTP2_HPACK_HUFFMAN_HPACK_HUFFMAN_ENCODER_H_
// Functions supporting the encoding of strings using the HPACK-defined Huffman
// table.
#include <cstddef> // For size_t
#include "net/third_party/http2/platform/api/http2_export.h"
#include "net/third_party/http2/platform/api/http2_string.h"
#include "net/third_party/http2/platform/api/http2_string_piece.h"
namespace http2 {
// Returns the size of the Huffman encoding of |plain|, which may be greater
// than plain.size(). Mostly present for testing.
HTTP2_EXPORT_PRIVATE size_t ExactHuffmanSize(Http2StringPiece plain);
// Returns the size of the Huffman encoding of |plain|, unless it is greater
// than or equal to plain.size(), in which case a value greater than or equal to
// plain.size() is returned. The advantage of this over ExactHuffmanSize is that
// it doesn't read as much of the input string in the event that the string is
// not compressible by HuffmanEncode (i.e. when the encoding is longer than the
// original string, it stops reading the input string as soon as it knows that).
HTTP2_EXPORT_PRIVATE size_t BoundedHuffmanSize(Http2StringPiece plain);
// Encode the plain text string |plain| with the Huffman encoding defined in
// the HPACK RFC, 7541. |*huffman| does not have to be empty, it is cleared at
// the beginning of this function. This allows reusing the same string object
// across multiple invocations.
HTTP2_EXPORT_PRIVATE void HuffmanEncode(Http2StringPiece plain,
Http2String* huffman);
} // namespace http2
#endif // NET_THIRD_PARTY_HTTP2_HPACK_HUFFMAN_HPACK_HUFFMAN_DECODER_H_
#include "net/third_party/http2/hpack/huffman/hpack_huffman_encoder.h"
#include "base/macros.h"
#include "base/stl_util.h"
#include "net/third_party/http2/platform/api/http2_string_utils.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace http2 {
namespace {
TEST(HuffmanEncoderTest, SpecRequestExamples) {
Http2String test_table[] = {
Http2HexDecode("f1e3c2e5f23a6ba0ab90f4ff"),
"www.example.com",
Http2HexDecode("a8eb10649cbf"),
"no-cache",
Http2HexDecode("25a849e95ba97d7f"),
"custom-key",
Http2HexDecode("25a849e95bb8e8b4bf"),
"custom-value",
};
for (size_t i = 0; i != base::size(test_table); i += 2) {
const Http2String& huffman_encoded(test_table[i]);
const Http2String& plain_string(test_table[i + 1]);
EXPECT_EQ(ExactHuffmanSize(plain_string), huffman_encoded.size());
EXPECT_EQ(BoundedHuffmanSize(plain_string), huffman_encoded.size());
Http2String buffer;
buffer.reserve();
HuffmanEncode(plain_string, &buffer);
EXPECT_EQ(buffer, huffman_encoded) << "Error encoding " << plain_string;
}
}
TEST(HuffmanEncoderTest, SpecResponseExamples) {
// clang-format off
Http2String test_table[] = {
Http2HexDecode("6402"),
"302",
Http2HexDecode("aec3771a4b"),
"private",
Http2HexDecode("d07abe941054d444a8200595040b8166"
"e082a62d1bff"),
"Mon, 21 Oct 2013 20:13:21 GMT",
Http2HexDecode("9d29ad171863c78f0b97c8e9ae82ae43"
"d3"),
"https://www.example.com",
Http2HexDecode("94e7821dd7f2e6c7b335dfdfcd5b3960"
"d5af27087f3672c1ab270fb5291f9587"
"316065c003ed4ee5b1063d5007"),
"foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1",
};
// clang-format on
for (size_t i = 0; i != base::size(test_table); i += 2) {
const Http2String& huffman_encoded(test_table[i]);
const Http2String& plain_string(test_table[i + 1]);
EXPECT_EQ(ExactHuffmanSize(plain_string), huffman_encoded.size());
EXPECT_EQ(BoundedHuffmanSize(plain_string), huffman_encoded.size());
Http2String buffer;
buffer.reserve(huffman_encoded.size());
const size_t capacity = buffer.capacity();
HuffmanEncode(plain_string, &buffer);
EXPECT_EQ(buffer, huffman_encoded) << "Error encoding " << plain_string;
EXPECT_EQ(capacity, buffer.capacity());
}
}
TEST(HuffmanEncoderTest, EncodedSizeAgreesWithEncodeString) {
Http2String test_table[] = {
"",
"Mon, 21 Oct 2013 20:13:21 GMT",
"https://www.example.com",
"foo=ASDJKHQKBZXOQWEOPIUAXQWEOIU; max-age=3600; version=1",
Http2String(1, '\0'),
Http2String("foo\0bar", 7),
Http2String(256, '\0'),
};
// Modify last |test_table| entry to cover all codes.
for (size_t i = 0; i != 256; ++i) {
test_table[base::size(test_table) - 1][i] = static_cast<char>(i);
}
for (size_t i = 0; i != base::size(test_table); ++i) {
const Http2String& plain_string = test_table[i];
Http2String huffman_encoded;
HuffmanEncode(plain_string, &huffman_encoded);
EXPECT_EQ(huffman_encoded.size(), ExactHuffmanSize(plain_string));
EXPECT_LE(BoundedHuffmanSize(plain_string), plain_string.size());
EXPECT_LE(BoundedHuffmanSize(plain_string), ExactHuffmanSize(plain_string));
}
}
} // namespace
} // namespace http2
// A test of roundtrips through the encoder and decoder.
#include <stddef.h>
#include "net/third_party/http2/decoder/decode_buffer.h"
#include "net/third_party/http2/decoder/decode_status.h"
#include "net/third_party/http2/hpack/huffman/hpack_huffman_decoder.h"
#include "net/third_party/http2/hpack/huffman/hpack_huffman_encoder.h"
#include "net/third_party/http2/platform/api/http2_string.h"
#include "net/third_party/http2/platform/api/http2_string_piece.h"
#include "net/third_party/http2/platform/api/http2_string_utils.h"
#include "net/third_party/http2/platform/api/random_util_helper.h"
#include "net/third_party/http2/tools/random_decoder_test.h"
#include "testing/gtest/include/gtest/gtest.h"
using ::testing::AssertionResult;
using ::testing::AssertionSuccess;
using ::testing::tuple;
namespace http2 {
namespace test {
namespace {
Http2String GenAsciiNonControlSet() {
Http2String s;
const char space = ' '; // First character after the control characters: 0x20
const char del = 127; // First character after the non-control characters.
for (char c = space; c < del; ++c) {
s.push_back(c);
}
return s;
}
class HpackHuffmanTranscoderTest : public RandomDecoderTest {
protected:
HpackHuffmanTranscoderTest()
: ascii_non_control_set_(GenAsciiNonControlSet()) {
// The decoder may return true, and its accumulator may be empty, at
// many boundaries while decoding, and yet the whole string hasn't
// been decoded.
stop_decode_on_done_ = false;
}
DecodeStatus StartDecoding(DecodeBuffer* b) override {
input_bytes_seen_ = 0;
output_buffer_.clear();
decoder_.Reset();
return ResumeDecoding(b);
}
DecodeStatus ResumeDecoding(DecodeBuffer* b) override {
input_bytes_seen_ += b->Remaining();
Http2StringPiece sp(b->cursor(), b->Remaining());
if (decoder_.Decode(sp, &output_buffer_)) {
b->AdvanceCursor(b->Remaining());
// Successfully decoded (or buffered) the bytes in Http2StringPiece.
EXPECT_LE(input_bytes_seen_, input_bytes_expected_);
// Have we reached the end of the encoded string?
if (input_bytes_expected_ == input_bytes_seen_) {
if (decoder_.InputProperlyTerminated()) {
return DecodeStatus::kDecodeDone;
} else {
return DecodeStatus::kDecodeError;
}
}
return DecodeStatus::kDecodeInProgress;
}
return DecodeStatus::kDecodeError;
}
AssertionResult TranscodeAndValidateSeveralWays(
Http2StringPiece plain,
Http2StringPiece expected_huffman) {
Http2String encoded;
HuffmanEncode(plain, &encoded);
if (expected_huffman.size() > 0 || plain.empty()) {
VERIFY_EQ(encoded, expected_huffman);
}
input_bytes_expected_ = encoded.size();
auto validator = [plain, this]() -> AssertionResult {
VERIFY_EQ(output_buffer_.size(), plain.size());
VERIFY_EQ(output_buffer_, plain);
return AssertionSuccess();
};
DecodeBuffer db(encoded);
bool return_non_zero_on_first = false;
return DecodeAndValidateSeveralWays(&db, return_non_zero_on_first,
ValidateDoneAndEmpty(validator));
}
AssertionResult TranscodeAndValidateSeveralWays(Http2StringPiece plain) {
return TranscodeAndValidateSeveralWays(plain, "");
}
Http2String RandomAsciiNonControlString(int length) {
return RandomString(RandomPtr(), length, ascii_non_control_set_);
}
Http2String RandomBytes(int length) { return Random().RandString(length); }
const Http2String ascii_non_control_set_;
HpackHuffmanDecoder decoder_;
Http2String output_buffer_;
size_t input_bytes_seen_;
size_t input_bytes_expected_;
};
TEST_F(HpackHuffmanTranscoderTest, RoundTripRandomAsciiNonControlString) {
for (size_t length = 0; length != 20; length++) {
const Http2String s = RandomAsciiNonControlString(length);
ASSERT_TRUE(TranscodeAndValidateSeveralWays(s))
<< "Unable to decode:\n\n"
<< Http2HexDump(s) << "\n\noutput_buffer_:\n"
<< Http2HexDump(output_buffer_);
}
}
TEST_F(HpackHuffmanTranscoderTest, RoundTripRandomBytes) {
for (size_t length = 0; length != 20; length++) {
const Http2String s = RandomBytes(length);
ASSERT_TRUE(TranscodeAndValidateSeveralWays(s))
<< "Unable to decode:\n\n"
<< Http2HexDump(s) << "\n\noutput_buffer_:\n"
<< Http2HexDump(output_buffer_);
}
}
// Two parameters: decoder choice, and the character to round-trip.
class HpackHuffmanTranscoderAdjacentCharTest
: public HpackHuffmanTranscoderTest,
public ::testing::WithParamInterface<int> {
protected:
HpackHuffmanTranscoderAdjacentCharTest()
: c_(static_cast<char>(GetParam())) {}
const char c_;
};
INSTANTIATE_TEST_CASE_P(HpackHuffmanTranscoderAdjacentCharTest,
HpackHuffmanTranscoderAdjacentCharTest,
::testing::Range(0, 256));
// Test c_ adjacent to every other character, both before and after.
TEST_P(HpackHuffmanTranscoderAdjacentCharTest, RoundTripAdjacentChar) {
Http2String s;
for (int a = 0; a < 256; ++a) {
s.push_back(static_cast<char>(a));
s.push_back(c_);
s.push_back(static_cast<char>(a));
}
ASSERT_TRUE(TranscodeAndValidateSeveralWays(s));
}
// Two parameters: character to repeat, number of repeats.
class HpackHuffmanTranscoderRepeatedCharTest
: public HpackHuffmanTranscoderTest,
public ::testing::WithParamInterface<tuple<int, int>> {
protected:
HpackHuffmanTranscoderRepeatedCharTest()
: c_(static_cast<char>(::testing::get<0>(GetParam()))),
length_(::testing::get<1>(GetParam())) {}
Http2String MakeString() { return Http2String(length_, c_); }
private:
const char c_;
const size_t length_;
};
INSTANTIATE_TEST_CASE_P(
HpackHuffmanTranscoderRepeatedCharTest,
HpackHuffmanTranscoderRepeatedCharTest,
::testing::Combine(::testing::Range(0, 256),
::testing::Values(1, 2, 3, 4, 8, 16, 32)));
TEST_P(HpackHuffmanTranscoderRepeatedCharTest, RoundTripRepeatedChar) {
ASSERT_TRUE(TranscodeAndValidateSeveralWays(MakeString()));
}
} // namespace
} // namespace test
} // namespace http2
This diff is collapsed.
#ifndef NET_THIRD_PARTY_HTTP2_HPACK_HUFFMAN_HUFFMAN_SPEC_TABLES_H_
#define NET_THIRD_PARTY_HTTP2_HPACK_HUFFMAN_HUFFMAN_SPEC_TABLES_H_
// Tables describing the Huffman encoding of bytes as specified by RFC7541.
#include <cstdint>
namespace http2 {
struct HuffmanSpecTables {
// Number of bits in the encoding of each symbol (byte).
static const uint8_t kCodeLengths[257];
// The encoding of each symbol, right justified (as printed), which means that
// the last bit of the encoding is the low-order bit of the uint32.
static const uint32_t kRightCodes[257];
};
} // namespace http2
#endif // NET_THIRD_PARTY_HTTP2_HPACK_HUFFMAN_HUFFMAN_SPEC_TABLES_H_
#ifndef NET_THIRD_PARTY_HTTP2_PLATFORM_API_RANDOM_UTIL_HELPER_H_
#define NET_THIRD_PARTY_HTTP2_PLATFORM_API_RANDOM_UTIL_HELPER_H_
#include "net/third_party/http2/platform/impl/random_util_helper_impl.h"
namespace http2 {
namespace test {
class RandomBase;
inline Http2String RandomString(RandomBase* random,
int len,
Http2StringPiece alphabet) {
return RandomStringImpl(random, len, alphabet);
}
} // namespace test
} // namespace http2
#endif // NET_THIRD_PARTY_HTTP2_PLATFORM_API_RANDOM_UTIL_HELPER_H_
#include "net/third_party/http2/platform/impl/random_util_helper_impl.h"
namespace http2 {
namespace test {
Http2String RandomStringImpl(RandomBase* random,
int len,
Http2StringPiece alphabet) {
Http2String random_string;
random_string.reserve(len);
for (int i = 0; i < len; ++i)
random_string.push_back(alphabet[random->Uniform(alphabet.size())]);
return random_string;
}
} // namespace test
} // namespace http2
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef NET_THIRD_PARTY_HTTP2_PLATFORM_IMPL_RANDOM_UTIL_HELPER_IMPL_H_
#define NET_THIRD_PARTY_HTTP2_PLATFORM_IMPL_RANDOM_UTIL_HELPER_IMPL_H_
#include "net/third_party/http2/platform/api/http2_string.h"
#include "net/third_party/http2/platform/api/http2_string_piece.h"
#include "net/third_party/http2/tools/http2_random.h"
namespace http2 {
namespace test {
Http2String RandomStringImpl(RandomBase* random,
int len,
Http2StringPiece alphabet);
} // namespace test
} // namespace http2
#endif // NET_THIRD_PARTY_HTTP2_PLATFORM_IMPL_RANDOM_UTIL_HELPER_IMPL_H_
......@@ -7,6 +7,7 @@
#include <cmath>
#include "base/rand_util.h"
#include "net/third_party/http2/platform/api/random_util_helper.h"
#include "net/third_party/http2/tools/http2_random.h"
namespace http2 {
......@@ -33,14 +34,6 @@ void GenerateRandomSizeSkewedLowHelper(size_t max, size_t* x, size_t* y) {
} // anonymous namespace
Http2String RandomString(RandomBase* rng, int len, Http2StringPiece alphabet) {
Http2String random_string;
random_string.reserve(len);
for (int i = 0; i < len; ++i)
random_string.push_back(alphabet[rng->Uniform(alphabet.size())]);
return random_string;
}
size_t GenerateUniformInRange(size_t lo, size_t hi, RandomBase* rng) {
if (lo + 1 >= hi) {
return lo;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment