Commit 683a3487 authored by Findit's avatar Findit

Revert "Export the quic batch writer code to Chromium. Note it's not used...

Revert "Export the quic batch writer code to Chromium. Note it's not used anywhere in Chrome, we are just exporting the code so people outside of Google can take a look."

This reverts commit cc75805c.

Reason for revert:

Findit (https://goo.gl/kROfz5) identified CL at revision 605652 as the
culprit for failures in the build cycles as shown on:
https://findit-for-me.appspot.com/waterfall/culprit?key=ag9zfmZpbmRpdC1mb3ItbWVyRAsSDVdmU3VzcGVjdGVkQ0wiMWNocm9taXVtL2NjNzU4MDVjMWYzMmNhZDY2MDFmNzVjZjRjYWRmYmVlMjM0NDYzN2QM

Sample Failed Build: https://ci.chromium.org/buildbot/chromium.memory/Linux%20MSan%20Tests/12757

Sample Failed Step: net_unittests

Original change's description:
> Export the quic batch writer code to Chromium. Note it's not used anywhere in Chrome, we are just exporting the code so people outside of Google can take a look.
> 
> Change-Id: I9fd9486172a13cd8924dd622aa6984285e07ac6c
> Reviewed-on: https://chromium-review.googlesource.com/c/1315961
> Reviewed-by: Ryan Hamilton <rch@chromium.org>
> Commit-Queue: Bin Wu <wub@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#605652}

Change-Id: I4f8a4dea97f700bf98c8a244f9a505f083eb53b1
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Reviewed-on: https://chromium-review.googlesource.com/c/1319850
Cr-Commit-Position: refs/heads/master@{#605679}
parent f68d96be
......@@ -2974,18 +2974,8 @@ if (is_linux) {
"third_party/quic/core/quic_epoll_connection_helper.h",
"third_party/quic/core/quic_packet_reader.cc",
"third_party/quic/core/quic_packet_reader.h",
"third_party/quic/platform/impl/batch_writer/quic_batch_writer_base.cc",
"third_party/quic/platform/impl/batch_writer/quic_batch_writer_base.h",
"third_party/quic/platform/impl/batch_writer/quic_batch_writer_buffer.cc",
"third_party/quic/platform/impl/batch_writer/quic_batch_writer_buffer.h",
"third_party/quic/platform/impl/batch_writer/quic_gso_batch_writer.cc",
"third_party/quic/platform/impl/batch_writer/quic_gso_batch_writer.h",
"third_party/quic/platform/impl/batch_writer/quic_sendmmsg_batch_writer.cc",
"third_party/quic/platform/impl/batch_writer/quic_sendmmsg_batch_writer.h",
"third_party/quic/platform/impl/quic_epoll_clock.cc",
"third_party/quic/platform/impl/quic_epoll_clock.h",
"third_party/quic/platform/impl/quic_linux_socket_utils.cc",
"third_party/quic/platform/impl/quic_linux_socket_utils.h",
"third_party/quic/platform/impl/quic_socket_utils.cc",
"third_party/quic/platform/impl/quic_socket_utils.h",
"third_party/quic/tools/quic_client.cc",
......@@ -5359,12 +5349,7 @@ test("net_unittests") {
"third_party/quic/core/quic_epoll_alarm_factory_test.cc",
"third_party/quic/core/quic_epoll_connection_helper_test.cc",
"third_party/quic/core/stateless_rejector_test.cc",
"third_party/quic/platform/impl/batch_writer/quic_batch_writer_buffer_test.cc",
"third_party/quic/platform/impl/batch_writer/quic_batch_writer_test.h",
"third_party/quic/platform/impl/batch_writer/quic_gso_batch_writer_test.cc",
"third_party/quic/platform/impl/batch_writer/quic_sendmmsg_batch_writer_test.cc",
"third_party/quic/platform/impl/quic_epoll_clock_test.cc",
"third_party/quic/platform/impl/quic_linux_socket_utils_test.cc",
"third_party/quic/platform/impl/quic_socket_utils_test.cc",
"third_party/quic/tools/quic_client_test.cc",
"third_party/quic/tools/quic_memory_cache_backend_test.cc",
......
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_base.h"
#include "net/third_party/quic/platform/api/quic_ptr_util.h"
namespace quic {
QuicBatchWriterBase::QuicBatchWriterBase(
std::unique_ptr<QuicBatchWriterBuffer> batch_buffer)
: write_blocked_(false), batch_buffer_(std::move(batch_buffer)) {}
WriteResult QuicBatchWriterBase::WritePacket(
const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
PerPacketOptions* options) {
const WriteResult result =
InternalWritePacket(buffer, buf_len, self_address, peer_address, options);
if (result.status == WRITE_STATUS_BLOCKED) {
write_blocked_ = true;
}
return result;
}
WriteResult QuicBatchWriterBase::InternalWritePacket(
const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
PerPacketOptions* options) {
if (buf_len > kMaxPacketSize) {
return WriteResult(WRITE_STATUS_MSG_TOO_BIG, EMSGSIZE);
}
const CanBatchResult can_batch_result =
CanBatch(buffer, buf_len, self_address, peer_address, options);
bool buffered = false;
bool flush = can_batch_result.must_flush;
if (can_batch_result.can_batch) {
QuicBatchWriterBuffer::PushResult push_result =
batch_buffer_->PushBufferedWrite(buffer, buf_len, self_address,
peer_address, options);
if (push_result.succeeded) {
buffered = true;
// If there's no space left after the packet is buffered, force a flush.
flush = flush || (batch_buffer_->GetNextWriteLocation() == nullptr);
} else {
// If there's no space without this packet, force a flush.
flush = true;
}
}
if (!flush) {
return WriteResult(WRITE_STATUS_OK, 0);
}
size_t num_buffered_packets = buffered_writes().size();
const FlushImplResult flush_result = CheckedFlush();
const WriteResult& result = flush_result.write_result;
QUIC_DVLOG(1) << "Internally flushed " << flush_result.num_packets_sent
<< " out of " << num_buffered_packets
<< " packets. WriteResult=" << result;
if (result.status != WRITE_STATUS_OK) {
if (result.status == WRITE_STATUS_BLOCKED && buffered) {
// Return OK if buffered successfully but write blocked while flush. The
// caller should handle write blockage by checking if IsWriteBlocked().
return WriteResult(WRITE_STATUS_OK, 0);
}
return result;
}
if (!buffered) {
QuicBatchWriterBuffer::PushResult push_result =
batch_buffer_->PushBufferedWrite(buffer, buf_len, self_address,
peer_address, options);
buffered = push_result.succeeded;
// Since buffered_writes has been emptied, this write must have been
// buffered successfully.
QUIC_BUG_IF(!buffered) << "Failed to push to an empty batch buffer."
<< " self_addr:" << self_address.ToString()
<< ", peer_addr:" << peer_address.ToString()
<< ", buf_len:" << buf_len;
}
return result;
}
QuicBatchWriterBase::FlushImplResult QuicBatchWriterBase::CheckedFlush() {
if (buffered_writes().empty()) {
return FlushImplResult{WriteResult(WRITE_STATUS_OK, 0),
/*num_packets_sent=*/0, /*bytes_written=*/0};
}
const FlushImplResult flush_result = FlushImpl();
// Either flush_result.write_result.status is not WRITE_STATUS_OK, or it is
// WRITE_STATUS_OK and batch_buffer is empty.
DCHECK(flush_result.write_result.status != WRITE_STATUS_OK ||
buffered_writes().empty());
return flush_result;
}
WriteResult QuicBatchWriterBase::Flush() {
size_t num_buffered_packets = buffered_writes().size();
const FlushImplResult flush_result = CheckedFlush();
QUIC_DVLOG(1) << "Externally flushed " << flush_result.num_packets_sent
<< " out of " << num_buffered_packets
<< " packets. WriteResult=" << flush_result.write_result;
if (flush_result.write_result.status == WRITE_STATUS_BLOCKED) {
write_blocked_ = true;
}
return flush_result.write_result;
}
} // namespace quic
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_BATCH_WRITER_BASE_H_
#define NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_BATCH_WRITER_BASE_H_
#include "net/third_party/quic/core/quic_packet_writer.h"
#include "net/third_party/quic/core/quic_types.h"
#include "net/third_party/quic/platform/api/quic_ip_address.h"
#include "net/third_party/quic/platform/api/quic_socket_address.h"
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_buffer.h"
namespace quic {
// QuicBatchWriterBase implements logic common to all derived batch writers,
// including maintaining write blockage state and a skeleton implemention of
// WritePacket().
// A derived batch writer must override the FlushImpl() function to send all
// buffered writes in a batch. It must also override the CanBatch() function
// to control whether/when a WritePacket() call should flush.
class QuicBatchWriterBase : public QuicPacketWriter {
public:
explicit QuicBatchWriterBase(
std::unique_ptr<QuicBatchWriterBuffer> batch_buffer);
WriteResult WritePacket(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
PerPacketOptions* options) override;
bool IsWriteBlockedDataBuffered() const final { return false; }
bool IsWriteBlocked() const final { return write_blocked_; }
void SetWritable() final { write_blocked_ = false; }
QuicByteCount GetMaxPacketSize(
const QuicSocketAddress& peer_address) const final {
return kMaxPacketSize;
}
bool SupportsReleaseTime() const final { return false; }
bool IsBatchMode() const final { return true; }
char* GetNextWriteLocation(const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address) final {
return batch_buffer_->GetNextWriteLocation();
}
WriteResult Flush() final;
protected:
const QuicBatchWriterBuffer& batch_buffer() const { return *batch_buffer_; }
QuicBatchWriterBuffer& batch_buffer() { return *batch_buffer_; }
const QuicDeque<BufferedWrite>& buffered_writes() const {
return batch_buffer_->buffered_writes();
}
struct CanBatchResult {
CanBatchResult(bool can_batch, bool must_flush)
: can_batch(can_batch), must_flush(must_flush) {}
// Whether this write can be batched with existing buffered writes.
bool can_batch;
// If |can_batch|, whether the caller must flush after this packet is
// buffered.
// Always true if not |can_batch|.
bool must_flush;
};
// Given the existing buffered writes(in buffered_writes()), whether a new
// write(in the arguments) can be batched.
virtual CanBatchResult CanBatch(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* options) const = 0;
struct FlushImplResult {
// The return value of the Flush() interface, which is:
// - WriteResult(WRITE_STATUS_OK, <bytes_flushed>) if all buffered writes
// were sent successfully.
// - WRITE_STATUS_BLOCKED or WRITE_STATUS_ERROR, if the batch write is
// blocked or returned an error while sending. If a portion of buffered
// writes were sent successfully, |FlushImplResult.num_packets_sent| and
// |FlushImplResult.bytes_written| contain the number of successfully sent
// packets and their total bytes.
WriteResult write_result;
int num_packets_sent;
// If write_result.status == WRITE_STATUS_OK, |bytes_written| will be equal
// to write_result.bytes_written. Otherwise |bytes_written| will be the
// number of bytes written before WRITE_BLOCK or WRITE_ERROR happened.
int bytes_written;
};
// Send all buffered writes(in buffered_writes()) in a batch.
// buffered_writes() is guaranteed to be non-empty when this function is
// called.
virtual FlushImplResult FlushImpl() = 0;
private:
WriteResult InternalWritePacket(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
PerPacketOptions* options);
// Calls FlushImpl() and check its post condition.
FlushImplResult CheckedFlush();
bool write_blocked_;
std::unique_ptr<QuicBatchWriterBuffer> batch_buffer_;
};
// QuicUdpBatchWriter is a batch writer backed by a UDP socket.
class QuicUdpBatchWriter : public QuicBatchWriterBase {
public:
QuicUdpBatchWriter(std::unique_ptr<QuicBatchWriterBuffer> batch_buffer,
int fd)
: QuicBatchWriterBase(std::move(batch_buffer)), fd_(fd) {}
int fd() const { return fd_; }
private:
const int fd_;
};
} // namespace quic
#endif // NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_BATCH_WRITER_BASE_H_
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_buffer.h"
#include <sstream>
namespace quic {
QuicBatchWriterBuffer::QuicBatchWriterBuffer() {}
QuicString QuicBatchWriterBuffer::DebugString() const {
std::ostringstream os;
os << "{ buffer: " << static_cast<const void*>(buffer_)
<< " buffer_end: " << static_cast<const void*>(buffer_end())
<< " buffered_writes_.size(): " << buffered_writes_.size()
<< " next_write_loc: " << static_cast<const void*>(GetNextWriteLocation())
<< " SizeInUse: " << SizeInUse() << " }";
return os.str();
}
bool QuicBatchWriterBuffer::Invariants() const {
// Buffers in buffered_writes_ should not overlap, and collectively they
// should cover a continuous prefix of buffer_.
const char* next_buffer = buffer_;
for (auto iter = buffered_writes_.begin(); iter != buffered_writes_.end();
++iter) {
if ((iter->buffer != next_buffer) ||
(iter->buffer + iter->buf_len > buffer_end())) {
return false;
}
next_buffer += iter->buf_len;
}
return (next_buffer - buffer_) == static_cast<int>(SizeInUse());
}
char* QuicBatchWriterBuffer::GetNextWriteLocation() const {
const char* next_loc =
buffered_writes_.empty()
? buffer_
: buffered_writes_.back().buffer + buffered_writes_.back().buf_len;
if (buffer_end() - next_loc < static_cast<int>(kMaxPacketSize)) {
return nullptr;
}
return const_cast<char*>(next_loc);
}
QuicBatchWriterBuffer::PushResult QuicBatchWriterBuffer::PushBufferedWrite(
const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* options) {
DCHECK(Invariants());
DCHECK_LE(buf_len, kMaxPacketSize);
PushResult result = {/*succeeded=*/false, /*buffer_copied=*/false};
char* next_write_location = GetNextWriteLocation();
if (next_write_location == nullptr) {
return result;
}
if (buffer != next_write_location) {
if (IsExternalBuffer(buffer, buf_len)) {
memcpy(next_write_location, buffer, buf_len);
} else if (IsInternalBuffer(buffer, buf_len)) {
memmove(next_write_location, buffer, buf_len);
} else {
QUIC_BUG << "Buffer[" << static_cast<const void*>(buffer) << ", "
<< static_cast<const void*>(buffer + buf_len)
<< ") overlaps with internal buffer["
<< static_cast<const void*>(buffer_) << ", "
<< static_cast<const void*>(buffer_end()) << ")";
return result;
}
result.buffer_copied = true;
} else {
// In place push, do nothing.
}
buffered_writes_.emplace_back(
next_write_location, buf_len, self_address, peer_address,
options ? options->Clone() : std::unique_ptr<PerPacketOptions>());
DCHECK(Invariants());
result.succeeded = true;
return result;
}
QuicBatchWriterBuffer::PopResult QuicBatchWriterBuffer::PopBufferedWrite(
int32_t num_buffered_writes) {
DCHECK(Invariants());
DCHECK_GE(num_buffered_writes, 0);
DCHECK_LE(num_buffered_writes, static_cast<int>(buffered_writes_.size()));
PopResult result = {/*num_buffers_popped=*/0,
/*moved_remaining_buffers=*/false};
result.num_buffers_popped = std::max<int32_t>(num_buffered_writes, 0);
result.num_buffers_popped =
std::min<int32_t>(result.num_buffers_popped, buffered_writes_.size());
buffered_writes_.erase(buffered_writes_.begin(),
buffered_writes_.begin() + result.num_buffers_popped);
if (!buffered_writes_.empty()) {
// If not all buffered writes are erased, the remaining ones will not cover
// a continuous prefix of buffer_. We'll fix it by moving the remaining
// buffers to the beginning of buffer_ and adjust the buffer pointers in all
// remaining buffered writes.
// This should happen very rarely, about once per write block.
result.moved_remaining_buffers = true;
const char* buffer_before_move = buffered_writes_.begin()->buffer;
size_t buffer_len_to_move = buffered_writes_.rbegin()->buffer +
buffered_writes_.rbegin()->buf_len -
buffer_before_move;
memmove(buffer_, buffer_before_move, buffer_len_to_move);
size_t distance_to_move = buffer_before_move - buffer_;
for (BufferedWrite& buffered_write : buffered_writes_) {
buffered_write.buffer -= distance_to_move;
}
DCHECK_EQ(buffer_, buffered_writes_.begin()->buffer);
}
DCHECK(Invariants());
return result;
}
size_t QuicBatchWriterBuffer::SizeInUse() const {
if (buffered_writes_.empty()) {
return 0;
}
return buffered_writes_.back().buffer + buffered_writes_.back().buf_len -
buffer_;
}
} // namespace quic
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_BATCH_WRITER_BUFFER_H_
#define NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_BATCH_WRITER_BUFFER_H_
#include "net/third_party/quic/core/quic_packet_writer.h"
#include "net/third_party/quic/platform/api/quic_aligned.h"
#include "net/third_party/quic/platform/api/quic_containers.h"
#include "net/third_party/quic/platform/api/quic_ip_address.h"
#include "net/third_party/quic/platform/api/quic_socket_address.h"
#include "net/third_party/quic/platform/api/quic_string.h"
#include "net/third_party/quic/platform/impl/quic_linux_socket_utils.h"
namespace quic {
// QuicBatchWriterBuffer manages an internal buffer to hold data from multiple
// packets. Packet data are placed continuously within the internal buffer such
// that they can be sent by a QuicGsoBatchWriter.
// This class can also be used by a QuicBatchWriter which uses sendmmsg,
// although it is not optimized for that use case.
class QuicBatchWriterBuffer {
public:
QuicBatchWriterBuffer();
char* GetNextWriteLocation() const;
// Push a buffered write to the back.
struct PushResult {
bool succeeded;
// True in one of the following cases:
// 1) The packet buffer is external and copied to the internal buffer, or
// 2) The packet buffer is from the internal buffer and moved within it.
// This only happens if PopBufferedWrite is called in the middle of a
// in-place push.
// Only valid if |succeeded| is true.
bool buffer_copied;
};
PushResult PushBufferedWrite(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* options);
// Pop |num_buffered_writes| buffered writes from the front.
// |num_buffered_writes| will be capped to [0, buffered_writes().size()]
// before it is used.
struct PopResult {
int32_t num_buffers_popped;
// True if after |num_buffers_popped| buffers are popped from front, the
// remaining buffers are moved to the beginning of the internal buffer.
// This should normally be false.
bool moved_remaining_buffers;
};
PopResult PopBufferedWrite(int32_t num_buffered_writes);
const QuicDeque<BufferedWrite>& buffered_writes() const {
return buffered_writes_;
}
bool IsExternalBuffer(const char* buffer, size_t buf_len) const {
return (buffer + buf_len) <= buffer_ || buffer >= buffer_end();
}
bool IsInternalBuffer(const char* buffer, size_t buf_len) const {
return buffer >= buffer_ && (buffer + buf_len) <= buffer_end();
}
// Number of bytes used in |buffer_|.
// PushBufferedWrite() increases this; PopBufferedWrite decreases this.
size_t SizeInUse() const;
// Rounded up from |kMaxGsoPacketSize|, which is the maximum allowed
// size of a GSO packet.
static const size_t kBufferSize = 64 * 1024;
QuicString DebugString() const;
protected:
// Whether the invariants of the buffer are upheld. For debug & test only.
bool Invariants() const;
const char* buffer_end() const { return buffer_ + sizeof(buffer_); }
QUIC_CACHELINE_ALIGNED char buffer_[kBufferSize];
QuicDeque<BufferedWrite> buffered_writes_;
};
} // namespace quic
#endif // NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_BATCH_WRITER_BUFFER_H_
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/third_party/quic/platform/impl/batch_writer/quic_gso_batch_writer.h"
#include "net/third_party/quic/platform/impl/quic_linux_socket_utils.h"
namespace quic {
QuicGsoBatchWriter::QuicGsoBatchWriter(
std::unique_ptr<QuicBatchWriterBuffer> batch_buffer,
int fd)
: QuicUdpBatchWriter(std::move(batch_buffer), fd) {}
QuicGsoBatchWriter::CanBatchResult QuicGsoBatchWriter::CanBatch(
const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* /*options*/) const {
// If there is nothing buffered already, this write will be included in this
// batch.
if (buffered_writes().empty()) {
return CanBatchResult(/*can_batch=*/true, /*must_flush=*/false);
}
// The new write can be batched if all of the following are true:
// [0] The total number of the GSO segments(one write=one segment, including
// the new write) must not exceed |max_segments|.
// [1] It has the same source and destination addresses as already buffered
// writes.
// [2] It won't cause this batch to exceed kMaxGsoPacketSize.
// [3] Already buffered writes all have the same length.
// [4] Length of already buffered writes must >= length of the new write.
const BufferedWrite& first = buffered_writes().front();
const BufferedWrite& last = buffered_writes().back();
size_t max_segments = MaxSegments(first.buf_len);
bool can_batch =
buffered_writes().size() < max_segments && // [0]
last.self_address == self_address && // [1]
last.peer_address == peer_address && // [1]
batch_buffer().SizeInUse() + buf_len <= kMaxGsoPacketSize && // [2]
first.buf_len == last.buf_len && // [3]
first.buf_len >= buf_len; // [4]
// A flush is required if any of the following is true:
// [a] The new write can't be batched.
// [b] Length of the new write is different from the length of already
// buffered writes.
// [c] The total number of the GSO segments, including the new write, reaches
// |max_segments|.
bool must_flush = (!can_batch) || // [a]
(last.buf_len != buf_len) || // [b]
(buffered_writes().size() + 1 == max_segments); // [c]
return CanBatchResult(can_batch, must_flush);
}
// static
void QuicGsoBatchWriter::BuildCmsg(QuicMsgHdr* hdr,
const QuicIpAddress& self_address,
uint16_t gso_size) {
hdr->SetIpInNextCmsg(self_address);
if (gso_size > 0) {
*hdr->GetNextCmsgData<uint16_t>(SOL_UDP, UDP_SEGMENT) = gso_size;
}
}
QuicGsoBatchWriter::FlushImplResult QuicGsoBatchWriter::FlushImpl() {
return InternalFlushImpl<kCmsgSpace>(BuildCmsg);
}
} // namespace quic
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_GSO_BATCH_WRITER_H_
#define NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_GSO_BATCH_WRITER_H_
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_base.h"
namespace quic {
// QuicGsoBatchWriter sends QUIC packets in batches, using UDP socket's generic
// segmentation offload(GSO) capability.
class QuicGsoBatchWriter : public QuicUdpBatchWriter {
public:
QuicGsoBatchWriter(std::unique_ptr<QuicBatchWriterBuffer> batch_buffer,
int fd);
CanBatchResult CanBatch(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* options) const override;
FlushImplResult FlushImpl() override;
protected:
static size_t MaxSegments(size_t gso_size) {
// Max segments should be the min of UDP_MAX_SEGMENTS(64) and
// (((64KB - sizeof(ip hdr) - sizeof(udp hdr)) / MSS) + 1), in the typical
// case of IPv6 packets with 1500-byte MTU, the result is
// ((64KB - 40 - 8) / (1500 - 48)) + 1 = 46
// However, due a kernel bug, the limit is much lower for tiny gso_sizes.
return gso_size <= 2 ? 16 : 45;
}
static const int kCmsgSpace = kCmsgSpaceForIp + kCmsgSpaceForSegmentSize;
static void BuildCmsg(QuicMsgHdr* hdr,
const QuicIpAddress& self_address,
uint16_t gso_size);
template <size_t CmsgSpace, typename CmsgBuilderT>
FlushImplResult InternalFlushImpl(CmsgBuilderT cmsg_builder) {
DCHECK(!IsWriteBlocked());
DCHECK(!buffered_writes().empty());
FlushImplResult result = {WriteResult(WRITE_STATUS_OK, 0),
/*num_packets_sent=*/0, /*bytes_written=*/0};
WriteResult& write_result = result.write_result;
int total_bytes = batch_buffer().SizeInUse();
const BufferedWrite& first = buffered_writes().front();
char cbuf[CmsgSpace];
QuicMsgHdr hdr(first.buffer, total_bytes, first.peer_address, cbuf,
sizeof(cbuf));
uint16_t gso_size = buffered_writes().size() > 1 ? first.buf_len : 0;
cmsg_builder(&hdr, first.self_address, gso_size);
write_result = QuicSocketUtils::WritePacket(fd(), hdr);
QUIC_DVLOG(1) << "Write GSO packet result: " << write_result
<< ", fd: " << fd()
<< ", self_address: " << first.self_address.ToString()
<< ", peer_address: " << first.peer_address.ToString()
<< ", num_segments: " << buffered_writes().size()
<< ", total_bytes: " << total_bytes
<< ", gso_size: " << gso_size;
// All segments in a GSO packet share the same fate - if the write failed,
// none of them are sent, and it's not needed to call PopBufferedWrite().
if (write_result.status != WRITE_STATUS_OK) {
return result;
}
result.num_packets_sent = buffered_writes().size();
write_result.bytes_written = total_bytes;
result.bytes_written = total_bytes;
batch_buffer().PopBufferedWrite(buffered_writes().size());
QUIC_BUG_IF(!buffered_writes().empty())
<< "All packets should have been written on a successful return";
return result;
}
};
} // namespace quic
#endif // NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_GSO_BATCH_WRITER_H_
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/third_party/quic/platform/impl/batch_writer/quic_gso_batch_writer.h"
#include "net/third_party/quic/platform/api/quic_ptr_util.h"
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_test.h"
namespace quic {
namespace test {
namespace {
class QuicGsoBatchWriterIOTestDelegate
: public QuicUdpBatchWriterIOTestDelegate {
public:
bool ShouldSkip(const QuicUdpBatchWriterIOTestParams& params) override {
const QuicSocketAddress address =
params.address_family == AF_INET
? QuicSocketAddress(QuicIpAddress::Loopback4(), 0)
: QuicSocketAddress(QuicIpAddress::Loopback6(), 0);
bool overflow_supported = false;
int fd = QuicSocketUtils::CreateUDPSocket(
address, /*receive_buffer_size=*/kDefaultSocketReceiveBuffer,
/*send_buffer_size=*/kDefaultSocketReceiveBuffer, &overflow_supported);
if (fd < 0) {
QUIC_LOG(ERROR) << "CreateSocket() failed: " << strerror(errno);
return false; // Let the test fail rather than skip it.
}
const bool gso_not_supported =
QuicLinuxSocketUtils::GetUDPSegmentSize(fd) < 0;
close(fd);
if (gso_not_supported) {
QUIC_LOG(WARNING) << "Test skipped since GSO is not supported.";
return true;
}
QUIC_LOG(WARNING) << "OK: GSO is supported.";
return false;
}
void ResetWriter(int fd) override {
writer_ = QuicMakeUnique<QuicGsoBatchWriter>(
QuicMakeUnique<QuicBatchWriterBuffer>(), fd);
}
QuicUdpBatchWriter* GetWriter() override { return writer_.get(); }
private:
std::unique_ptr<QuicGsoBatchWriter> writer_;
};
INSTANTIATE_TEST_CASE_P(
QuicGsoBatchWriterTest,
QuicUdpBatchWriterIOTest,
testing::ValuesIn(
MakeQuicBatchWriterTestParams<QuicGsoBatchWriterIOTestDelegate>()));
class TestQuicGsoBatchWriter : public QuicGsoBatchWriter {
public:
using QuicGsoBatchWriter::batch_buffer;
using QuicGsoBatchWriter::CanBatch;
using QuicGsoBatchWriter::CanBatchResult;
using QuicGsoBatchWriter::MaxSegments;
using QuicGsoBatchWriter::QuicGsoBatchWriter;
};
// TestBufferedWrite is a copy-constructible BufferedWrite.
struct TestBufferedWrite : public BufferedWrite {
using BufferedWrite::BufferedWrite;
TestBufferedWrite(const TestBufferedWrite& other)
: BufferedWrite(other.buffer,
other.buf_len,
other.self_address,
other.peer_address,
other.options ? other.options->Clone()
: std::unique_ptr<PerPacketOptions>()) {}
};
// Pointed to by all instances of |BatchCriteriaTestData|. Content not used.
static char unused_packet_buffer[kMaxPacketSize];
struct BatchCriteriaTestData {
BatchCriteriaTestData(size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
bool can_batch,
bool must_flush)
: buffered_write(unused_packet_buffer,
buf_len,
self_address,
peer_address),
can_batch(can_batch),
must_flush(must_flush) {}
TestBufferedWrite buffered_write;
// Expected value of CanBatchResult.can_batch when batching |buffered_write|.
bool can_batch;
// Expected value of CanBatchResult.must_flush when batching |buffered_write|.
bool must_flush;
};
std::vector<BatchCriteriaTestData> BatchCriteriaTestData_SizeDecrease() {
const QuicIpAddress self_addr;
const QuicSocketAddress peer_addr;
std::vector<BatchCriteriaTestData> test_data_table = {
// clang-format off
// buf_len self_addr peer_addr can_batch must_flush
{1350, self_addr, peer_addr, true, false},
{1350, self_addr, peer_addr, true, false},
{1350, self_addr, peer_addr, true, false},
{39, self_addr, peer_addr, true, true},
{39, self_addr, peer_addr, false, true},
{1350, self_addr, peer_addr, false, true},
// clang-format on
};
return test_data_table;
}
std::vector<BatchCriteriaTestData> BatchCriteriaTestData_SizeIncrease() {
const QuicIpAddress self_addr;
const QuicSocketAddress peer_addr;
std::vector<BatchCriteriaTestData> test_data_table = {
// clang-format off
// buf_len self_addr peer_addr can_batch must_flush
{1350, self_addr, peer_addr, true, false},
{1350, self_addr, peer_addr, true, false},
{1350, self_addr, peer_addr, true, false},
{1351, self_addr, peer_addr, false, true},
// clang-format on
};
return test_data_table;
}
std::vector<BatchCriteriaTestData> BatchCriteriaTestData_AddressChange() {
const QuicIpAddress self_addr1 = QuicIpAddress::Loopback4();
const QuicIpAddress self_addr2 = QuicIpAddress::Loopback6();
const QuicSocketAddress peer_addr1(self_addr1, 666);
const QuicSocketAddress peer_addr2(self_addr1, 777);
const QuicSocketAddress peer_addr3(self_addr2, 666);
const QuicSocketAddress peer_addr4(self_addr2, 777);
std::vector<BatchCriteriaTestData> test_data_table = {
// clang-format off
// buf_len self_addr peer_addr can_batch must_flush
{1350, self_addr1, peer_addr1, true, false},
{1350, self_addr1, peer_addr1, true, false},
{1350, self_addr1, peer_addr1, true, false},
{1350, self_addr2, peer_addr1, false, true},
{1350, self_addr1, peer_addr2, false, true},
{1350, self_addr1, peer_addr3, false, true},
{1350, self_addr1, peer_addr4, false, true},
{1350, self_addr1, peer_addr4, false, true},
// clang-format on
};
return test_data_table;
}
std::vector<BatchCriteriaTestData> BatchCriteriaTestData_MaxSegments(
size_t gso_size) {
const QuicIpAddress self_addr;
const QuicSocketAddress peer_addr;
std::vector<BatchCriteriaTestData> test_data_table;
size_t max_segments = TestQuicGsoBatchWriter::MaxSegments(gso_size);
for (size_t i = 0; i < max_segments; ++i) {
bool is_last_in_batch = (i + 1 == max_segments);
test_data_table.push_back(
{gso_size, self_addr, peer_addr, true, is_last_in_batch});
}
test_data_table.push_back({gso_size, self_addr, peer_addr, false, true});
return test_data_table;
}
TEST(QuicGsoBatchWriterTest, BatchCriteria) {
std::unique_ptr<TestQuicGsoBatchWriter> writer;
std::vector<std::vector<BatchCriteriaTestData>> test_data_tables;
test_data_tables.emplace_back(BatchCriteriaTestData_SizeDecrease());
test_data_tables.emplace_back(BatchCriteriaTestData_SizeIncrease());
test_data_tables.emplace_back(BatchCriteriaTestData_AddressChange());
test_data_tables.emplace_back(BatchCriteriaTestData_MaxSegments(1));
test_data_tables.emplace_back(BatchCriteriaTestData_MaxSegments(2));
test_data_tables.emplace_back(BatchCriteriaTestData_MaxSegments(1350));
for (size_t i = 0; i < test_data_tables.size(); ++i) {
writer = QuicMakeUnique<TestQuicGsoBatchWriter>(
QuicMakeUnique<QuicBatchWriterBuffer>(), /*fd=*/-1);
const auto& test_data_table = test_data_tables[i];
for (size_t j = 0; j < test_data_table.size(); ++j) {
const BatchCriteriaTestData& test_data = test_data_table[j];
SCOPED_TRACE(testing::Message() << "i=" << i << ", j=" << j);
TestQuicGsoBatchWriter::CanBatchResult result = writer->CanBatch(
test_data.buffered_write.buffer, test_data.buffered_write.buf_len,
test_data.buffered_write.self_address,
test_data.buffered_write.peer_address,
/*options=*/nullptr);
ASSERT_EQ(test_data.can_batch, result.can_batch);
ASSERT_EQ(test_data.must_flush, result.must_flush);
if (result.can_batch) {
ASSERT_TRUE(
writer->batch_buffer()
.PushBufferedWrite(test_data.buffered_write.buffer,
test_data.buffered_write.buf_len,
test_data.buffered_write.self_address,
test_data.buffered_write.peer_address,
/*options=*/nullptr)
.succeeded);
}
}
}
}
} // namespace
} // namespace test
} // namespace quic
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/third_party/quic/platform/impl/batch_writer/quic_sendmmsg_batch_writer.h"
namespace quic {
QuicSendmmsgBatchWriter::QuicSendmmsgBatchWriter(
std::unique_ptr<QuicBatchWriterBuffer> batch_buffer,
int fd)
: QuicUdpBatchWriter(std::move(batch_buffer), fd) {}
QuicSendmmsgBatchWriter::CanBatchResult QuicSendmmsgBatchWriter::CanBatch(
const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* options) const {
return CanBatchResult(/*can_batch=*/true, /*must_flush=*/false);
}
QuicSendmmsgBatchWriter::FlushImplResult QuicSendmmsgBatchWriter::FlushImpl() {
return InternalFlushImpl(
kCmsgSpaceForIp,
[](QuicMMsgHdr* mhdr, int i, const BufferedWrite& buffered_write) {
mhdr->SetIpInNextCmsg(i, buffered_write.self_address);
});
}
QuicSendmmsgBatchWriter::FlushImplResult
QuicSendmmsgBatchWriter::InternalFlushImpl(size_t cmsg_space,
const CmsgBuilder& cmsg_builder) {
DCHECK(!IsWriteBlocked());
DCHECK(!buffered_writes().empty());
FlushImplResult result = {WriteResult(WRITE_STATUS_OK, 0),
/*num_packets_sent=*/0, /*bytes_written=*/0};
WriteResult& write_result = result.write_result;
auto first = buffered_writes().cbegin();
const auto last = buffered_writes().cend();
while (first != last) {
QuicMMsgHdr mhdr(first, last, cmsg_space, cmsg_builder);
int num_packets_sent;
write_result = QuicLinuxSocketUtils::WriteMultiplePackets(
fd(), &mhdr, &num_packets_sent);
QUIC_DVLOG(1) << "WriteMultiplePackets sent " << num_packets_sent
<< " out of " << mhdr.num_msgs()
<< " packets. WriteResult=" << write_result;
if (write_result.status != WRITE_STATUS_OK) {
DCHECK_EQ(0, num_packets_sent);
break;
} else if (num_packets_sent == 0) {
QUIC_BUG << "WriteMultiplePackets returned OK, but no packets were sent.";
write_result = WriteResult(WRITE_STATUS_ERROR, EIO);
break;
}
first += num_packets_sent;
result.num_packets_sent += num_packets_sent;
result.bytes_written += write_result.bytes_written;
}
// Call PopBufferedWrite() even if write_result.status is not WRITE_STATUS_OK,
// to deal with partial writes.
batch_buffer().PopBufferedWrite(result.num_packets_sent);
if (write_result.status != WRITE_STATUS_OK) {
return result;
}
QUIC_BUG_IF(!buffered_writes().empty())
<< "All packets should have been written on a successful return";
write_result.bytes_written = result.bytes_written;
return result;
}
} // namespace quic
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_SENDMMSG_BATCH_WRITER_H_
#define NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_SENDMMSG_BATCH_WRITER_H_
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_base.h"
#include "net/third_party/quic/platform/impl/quic_linux_socket_utils.h"
namespace quic {
class QuicSendmmsgBatchWriter : public QuicUdpBatchWriter {
public:
QuicSendmmsgBatchWriter(std::unique_ptr<QuicBatchWriterBuffer> batch_buffer,
int fd);
CanBatchResult CanBatch(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
const PerPacketOptions* options) const override;
FlushImplResult FlushImpl() override;
protected:
typedef QuicMMsgHdr::ControlBufferInitializer CmsgBuilder;
FlushImplResult InternalFlushImpl(size_t cmsg_space,
const CmsgBuilder& cmsg_builder);
};
} // namespace quic
#endif // NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_BATCH_WRITER_QUIC_SENDMMSG_BATCH_WRITER_H_
// Copyright (c) 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/third_party/quic/platform/impl/batch_writer/quic_sendmmsg_batch_writer.h"
#include "net/third_party/quic/platform/api/quic_ptr_util.h"
#include "net/third_party/quic/platform/impl/batch_writer/quic_batch_writer_test.h"
namespace quic {
namespace test {
namespace {
class QuicSendmmsgBatchWriterIOTestDelegate
: public QuicUdpBatchWriterIOTestDelegate {
public:
void ResetWriter(int fd) override {
writer_ = QuicMakeUnique<QuicSendmmsgBatchWriter>(
QuicMakeUnique<QuicBatchWriterBuffer>(), fd);
}
QuicUdpBatchWriter* GetWriter() override { return writer_.get(); }
private:
std::unique_ptr<QuicSendmmsgBatchWriter> writer_;
};
INSTANTIATE_TEST_CASE_P(
QuicSendmmsgBatchWriterTest,
QuicUdpBatchWriterIOTest,
testing::ValuesIn(MakeQuicBatchWriterTestParams<
QuicSendmmsgBatchWriterIOTestDelegate>()));
} // namespace
} // namespace test
} // namespace quic
#include "net/third_party/quic/platform/impl/quic_linux_socket_utils.h"
#include <netinet/in.h>
#include <iostream>
#include "net/third_party/quic/platform/api/quic_ip_address.h"
#include "net/third_party/quic/platform/api/quic_logging.h"
#include "net/third_party/quic/platform/api/quic_socket_address.h"
namespace quic {
void QuicMMsgHdr::InitOneHeader(int i, const BufferedWrite& buffered_write) {
mmsghdr* mhdr = GetMMsgHdr(i);
msghdr* hdr = &mhdr->msg_hdr;
iovec* iov = GetIov(i);
iov->iov_base = const_cast<char*>(buffered_write.buffer);
iov->iov_len = buffered_write.buf_len;
hdr->msg_iov = iov;
hdr->msg_iovlen = 1;
hdr->msg_control = nullptr;
hdr->msg_controllen = 0;
// Only support unconnected sockets.
DCHECK(buffered_write.peer_address.IsInitialized());
sockaddr_storage* peer_address_storage = GetPeerAddressStorage(i);
*peer_address_storage = buffered_write.peer_address.generic_address();
hdr->msg_name = peer_address_storage;
hdr->msg_namelen = peer_address_storage->ss_family == AF_INET
? sizeof(sockaddr_in)
: sizeof(sockaddr_in6);
}
void QuicMMsgHdr::SetIpInNextCmsg(int i, const QuicIpAddress& self_address) {
if (!self_address.IsInitialized()) {
return;
}
if (self_address.IsIPv4()) {
QuicSocketUtils::SetIpInfoInCmsgData(
self_address, GetNextCmsgData<in_pktinfo>(i, IPPROTO_IP, IP_PKTINFO));
} else {
QuicSocketUtils::SetIpInfoInCmsgData(
self_address,
GetNextCmsgData<in6_pktinfo>(i, IPPROTO_IPV6, IPV6_PKTINFO));
}
}
void* QuicMMsgHdr::GetNextCmsgDataInternal(int i,
int cmsg_level,
int cmsg_type,
size_t data_size) {
mmsghdr* mhdr = GetMMsgHdr(i);
msghdr* hdr = &mhdr->msg_hdr;
cmsghdr*& cmsg = *GetCmsgHdr(i);
// msg_controllen needs to be increased first, otherwise CMSG_NXTHDR will
// return nullptr.
hdr->msg_controllen += CMSG_SPACE(data_size);
DCHECK_LE(hdr->msg_controllen, cbuf_size_);
if (cmsg == nullptr) {
DCHECK_EQ(nullptr, hdr->msg_control);
hdr->msg_control = GetCbuf(i);
cmsg = CMSG_FIRSTHDR(hdr);
} else {
DCHECK_NE(nullptr, hdr->msg_control);
cmsg = CMSG_NXTHDR(hdr, cmsg);
}
DCHECK_NE(nullptr, cmsg) << "Insufficient control buffer space";
cmsg->cmsg_len = CMSG_LEN(data_size);
cmsg->cmsg_level = cmsg_level;
cmsg->cmsg_type = cmsg_type;
return CMSG_DATA(cmsg);
}
int QuicMMsgHdr::num_bytes_sent(int num_packets_sent) {
DCHECK_LE(0, num_packets_sent);
DCHECK_LE(num_packets_sent, num_msgs_);
int bytes_sent = 0;
iovec* iov = GetIov(0);
for (int i = 0; i < num_packets_sent; ++i) {
bytes_sent += iov[i].iov_len;
}
return bytes_sent;
}
// static
int QuicLinuxSocketUtils::GetUDPSegmentSize(int fd) {
int optval;
socklen_t optlen = sizeof(optval);
int rc = getsockopt(fd, SOL_UDP, UDP_SEGMENT, &optval, &optlen);
if (rc < 0) {
QUIC_LOG_EVERY_N_SEC(INFO, 10)
<< "getsockopt(UDP_SEGMENT) failed: " << strerror(errno);
return -1;
}
QUIC_LOG_EVERY_N_SEC(INFO, 10)
<< "getsockopt(UDP_SEGMENT) returned segment size: " << optval;
return optval;
}
} // namespace quic
#ifndef NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_QUIC_LINUX_SOCKET_UTILS_H_
#define NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_QUIC_LINUX_SOCKET_UTILS_H_
#include <errno.h>
#include <stddef.h>
#include <string.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <deque>
#include <functional>
#include <iterator>
#include <memory>
#include <type_traits>
#include <utility>
#include "net/third_party/quic/core/quic_packet_writer.h"
#include "net/third_party/quic/core/quic_types.h"
#include "net/third_party/quic/platform/api/quic_bug_tracker.h"
#include "net/third_party/quic/platform/api/quic_ip_address.h"
#include "net/third_party/quic/platform/api/quic_logging.h"
#include "net/third_party/quic/platform/api/quic_socket_address.h"
#include "net/third_party/quic/platform/impl/quic_socket_utils.h"
#ifndef SOL_UDP
#define SOL_UDP 17
#endif
#ifndef UDP_SEGMENT
#define UDP_SEGMENT 103
#endif
#ifndef UDP_MAX_SEGMENTS
#define UDP_MAX_SEGMENTS (1 << 6UL)
#endif
namespace quic {
// BufferedWrite holds all information needed to send a packet.
struct BufferedWrite {
BufferedWrite(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address)
: BufferedWrite(buffer,
buf_len,
self_address,
peer_address,
std::unique_ptr<PerPacketOptions>()) {}
BufferedWrite(const char* buffer,
size_t buf_len,
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address,
std::unique_ptr<PerPacketOptions> options)
: buffer(buffer),
buf_len(buf_len),
self_address(self_address),
peer_address(peer_address),
options(std::move(options)) {}
const char* buffer; // Not owned.
size_t buf_len;
QuicIpAddress self_address;
QuicSocketAddress peer_address;
std::unique_ptr<PerPacketOptions> options;
};
// QuicMMsgHdr is used to build mmsghdr objects that can be used to send
// multiple packets at once via ::sendmmsg.
//
// Example:
// QuicDeque<BufferedWrite> buffered_writes;
// ... (Populate buffered_writes) ...
//
// QuicMMsgHdr mhdr(
// buffered_writes.begin(), buffered_writes.end(), kCmsgSpaceForIp,
// [](QuicMMsgHdr* mhdr, int i, const BufferedWrite& buffered_write) {
// mhdr->SetIpInNextCmsg(i, buffered_write.self_address);
// });
//
// int num_packets_sent;
// QuicSocketUtils::WriteMultiplePackets(fd, &mhdr, &num_packets_sent);
class QuicMMsgHdr {
public:
typedef std::function<
void(QuicMMsgHdr* mhdr, int i, const BufferedWrite& buffered_write)>
ControlBufferInitializer;
template <typename IteratorT>
QuicMMsgHdr(const IteratorT& first,
const IteratorT& last,
size_t cbuf_size,
ControlBufferInitializer cbuf_initializer)
: num_msgs_(std::distance(first, last)), cbuf_size_(cbuf_size) {
static_assert(
std::is_same<typename std::iterator_traits<IteratorT>::value_type,
BufferedWrite>::value,
"Must iterate over a collection of BufferedWrite.");
DCHECK_LE(0, num_msgs_);
if (num_msgs_ == 0) {
return;
}
storage_.reset(new char[StorageSize()]);
bzero(&storage_[0], StorageSize());
int i = -1;
for (auto it = first; it != last; ++it) {
++i;
InitOneHeader(i, *it);
if (cbuf_initializer) {
cbuf_initializer(this, i, *it);
}
}
}
void SetIpInNextCmsg(int i, const QuicIpAddress& self_address);
template <typename DataType>
DataType* GetNextCmsgData(int i, int cmsg_level, int cmsg_type) {
return reinterpret_cast<DataType*>(
GetNextCmsgDataInternal(i, cmsg_level, cmsg_type, sizeof(DataType)));
}
mmsghdr* mhdr() { return GetMMsgHdr(0); }
int num_msgs() const { return num_msgs_; }
// Get the total number of bytes in the first |num_packets_sent| packets.
int num_bytes_sent(int num_packets_sent);
protected:
void InitOneHeader(int i, const BufferedWrite& buffered_write);
void* GetNextCmsgDataInternal(int i,
int cmsg_level,
int cmsg_type,
size_t data_size);
size_t StorageSize() const {
return num_msgs_ *
(sizeof(mmsghdr) + sizeof(iovec) + sizeof(sockaddr_storage) +
sizeof(cmsghdr*) + cbuf_size_);
}
mmsghdr* GetMMsgHdr(int i) {
auto* first = reinterpret_cast<mmsghdr*>(&storage_[0]);
return &first[i];
}
iovec* GetIov(int i) {
auto* first = reinterpret_cast<iovec*>(GetMMsgHdr(num_msgs_));
return &first[i];
}
sockaddr_storage* GetPeerAddressStorage(int i) {
auto* first = reinterpret_cast<sockaddr_storage*>(GetIov(num_msgs_));
return &first[i];
}
cmsghdr** GetCmsgHdr(int i) {
auto* first = reinterpret_cast<cmsghdr**>(GetPeerAddressStorage(num_msgs_));
return &first[i];
}
char* GetCbuf(int i) {
auto* first = reinterpret_cast<char*>(GetCmsgHdr(num_msgs_));
return &first[i * cbuf_size_];
}
const int num_msgs_;
// Size of cmsg buffer for each message.
const size_t cbuf_size_;
// storage_ holds the memory of
// |num_msgs_| mmsghdr
// |num_msgs_| iovec
// |num_msgs_| sockaddr_storage, for peer addresses
// |num_msgs_| cmsghdr*
// |num_msgs_| cbuf, each of size cbuf_size
std::unique_ptr<char[]> storage_;
};
// QuicSyscallWrapper wraps system calls for testing.
class QuicSyscallWrapper {
public:
int Sendmmsg(int sockfd,
mmsghdr* msgvec,
unsigned int vlen,
int flags) const {
return ::sendmmsg(sockfd, msgvec, vlen, flags);
}
};
class QuicLinuxSocketUtils : public QuicSocketUtils {
public:
// Return the UDP segment size of |fd|, 0 means segment size has not been set
// on this socket. If GSO is not supported, return -1.
static int GetUDPSegmentSize(int fd);
// Writes the packets in |mhdr| to the socket, using ::sendmmsg.
static WriteResult WriteMultiplePackets(int fd,
QuicMMsgHdr* mhdr,
int* num_packets_sent) {
return WriteMultiplePackets(fd, mhdr, num_packets_sent,
QuicSyscallWrapper());
}
template <typename SyscallWrapper>
static WriteResult WriteMultiplePackets(int fd,
QuicMMsgHdr* mhdr,
int* num_packets_sent,
const SyscallWrapper& syscall) {
*num_packets_sent = 0;
if (mhdr->num_msgs() <= 0) {
return WriteResult(WRITE_STATUS_ERROR, EINVAL);
}
int rc;
do {
rc = syscall.Sendmmsg(fd, mhdr->mhdr(), mhdr->num_msgs(), 0);
} while (rc < 0 && errno == EINTR);
if (rc > 0) {
*num_packets_sent = rc;
return WriteResult(WRITE_STATUS_OK, mhdr->num_bytes_sent(rc));
} else if (rc == 0) {
QUIC_BUG << "sendmmsg returned 0, returning WRITE_STATUS_ERROR. errno: "
<< errno;
errno = EIO;
}
return WriteResult((errno == EAGAIN || errno == EWOULDBLOCK)
? WRITE_STATUS_BLOCKED
: WRITE_STATUS_ERROR,
errno);
}
};
} // namespace quic
#endif // NET_THIRD_PARTY_QUIC_PLATFORM_IMPL_QUIC_LINUX_SOCKET_UTILS_H_
#include "net/third_party/quic/platform/impl/quic_linux_socket_utils.h"
#include <netinet/in.h>
#include <stdint.h>
#include <sstream>
#include <vector>
#include "net/third_party/quic/platform/api/quic_string.h"
#include "net/third_party/quic/platform/api/quic_test.h"
using testing::_;
using testing::InSequence;
using testing::Invoke;
namespace quic {
namespace test {
namespace {
class MockQuicSyscallWrapper {
public:
MOCK_CONST_METHOD4(
Sendmmsg,
int(int sockfd, mmsghdr* msgvec, unsigned int vlen, int flags));
};
class QuicLinuxSocketUtilsTest : public QuicTest {
protected:
WriteResult TestWriteMultiplePackets(
int fd,
const QuicDeque<BufferedWrite>::const_iterator& first,
const QuicDeque<BufferedWrite>::const_iterator& last,
int* num_packets_sent) {
QuicMMsgHdr mhdr(
first, last, kCmsgSpaceForIp,
[](QuicMMsgHdr* mhdr, int i, const BufferedWrite& buffered_write) {
mhdr->SetIpInNextCmsg(i, buffered_write.self_address);
});
WriteResult res = QuicLinuxSocketUtils::WriteMultiplePackets(
fd, &mhdr, num_packets_sent, mock_syscalls_);
return res;
}
MockQuicSyscallWrapper mock_syscalls_;
};
void CheckMsghdrWithoutCbuf(const msghdr* hdr,
const void* buffer,
size_t buf_len,
const QuicSocketAddress& peer_addr) {
EXPECT_EQ(
peer_addr.host().IsIPv4() ? sizeof(sockaddr_in) : sizeof(sockaddr_in6),
hdr->msg_namelen);
sockaddr_storage peer_generic_addr = peer_addr.generic_address();
EXPECT_EQ(0, memcmp(hdr->msg_name, &peer_generic_addr, hdr->msg_namelen));
EXPECT_EQ(1u, hdr->msg_iovlen);
EXPECT_EQ(buffer, hdr->msg_iov->iov_base);
EXPECT_EQ(buf_len, hdr->msg_iov->iov_len);
EXPECT_EQ(0, hdr->msg_flags);
EXPECT_EQ(nullptr, hdr->msg_control);
EXPECT_EQ(0u, hdr->msg_controllen);
}
void CheckIpAndGsoSizeInCbuf(msghdr* hdr,
const void* cbuf,
const QuicIpAddress& self_addr,
uint16_t gso_size) {
const bool is_ipv4 = self_addr.IsIPv4();
const size_t ip_cmsg_space = is_ipv4 ? kCmsgSpaceForIpv4 : kCmsgSpaceForIpv6;
EXPECT_EQ(cbuf, hdr->msg_control);
EXPECT_EQ(ip_cmsg_space + CMSG_SPACE(sizeof(uint16_t)), hdr->msg_controllen);
cmsghdr* cmsg = CMSG_FIRSTHDR(hdr);
EXPECT_EQ(cmsg->cmsg_len, is_ipv4 ? CMSG_LEN(sizeof(in_pktinfo))
: CMSG_LEN(sizeof(in6_pktinfo)));
EXPECT_EQ(cmsg->cmsg_level, is_ipv4 ? IPPROTO_IP : IPPROTO_IPV6);
EXPECT_EQ(cmsg->cmsg_type, is_ipv4 ? IP_PKTINFO : IPV6_PKTINFO);
const QuicString& self_addr_str = self_addr.ToPackedString();
if (is_ipv4) {
in_pktinfo* pktinfo = reinterpret_cast<in_pktinfo*>(CMSG_DATA(cmsg));
EXPECT_EQ(0, memcmp(&pktinfo->ipi_spec_dst, self_addr_str.c_str(),
self_addr_str.length()));
} else {
in6_pktinfo* pktinfo = reinterpret_cast<in6_pktinfo*>(CMSG_DATA(cmsg));
EXPECT_EQ(0, memcmp(&pktinfo->ipi6_addr, self_addr_str.c_str(),
self_addr_str.length()));
}
cmsg = CMSG_NXTHDR(hdr, cmsg);
EXPECT_EQ(cmsg->cmsg_len, CMSG_LEN(sizeof(uint16_t)));
EXPECT_EQ(cmsg->cmsg_level, SOL_UDP);
EXPECT_EQ(cmsg->cmsg_type, UDP_SEGMENT);
EXPECT_EQ(gso_size, *reinterpret_cast<uint16_t*>(CMSG_DATA(cmsg)));
EXPECT_EQ(nullptr, CMSG_NXTHDR(hdr, cmsg));
}
TEST_F(QuicLinuxSocketUtilsTest, QuicMMsgHdr) {
QuicDeque<BufferedWrite> buffered_writes;
char packet_buf1[1024];
char packet_buf2[512];
buffered_writes.emplace_back(
packet_buf1, sizeof(packet_buf1), QuicIpAddress::Loopback4(),
QuicSocketAddress(QuicIpAddress::Loopback4(), 4));
buffered_writes.emplace_back(
packet_buf2, sizeof(packet_buf2), QuicIpAddress::Loopback6(),
QuicSocketAddress(QuicIpAddress::Loopback6(), 6));
QuicMMsgHdr quic_mhdr_without_cbuf(buffered_writes.begin(),
buffered_writes.end(), 0, nullptr);
for (size_t i = 0; i < buffered_writes.size(); ++i) {
const BufferedWrite& bw = buffered_writes[i];
CheckMsghdrWithoutCbuf(&quic_mhdr_without_cbuf.mhdr()[i].msg_hdr, bw.buffer,
bw.buf_len, bw.peer_address);
}
QuicMMsgHdr quic_mhdr_with_cbuf(
buffered_writes.begin(), buffered_writes.end(),
kCmsgSpaceForIp + kCmsgSpaceForSegmentSize,
[](QuicMMsgHdr* mhdr, int i, const BufferedWrite& buffered_write) {
mhdr->SetIpInNextCmsg(i, buffered_write.self_address);
*mhdr->GetNextCmsgData<uint16_t>(i, SOL_UDP, UDP_SEGMENT) = 1300;
});
for (size_t i = 0; i < buffered_writes.size(); ++i) {
const BufferedWrite& bw = buffered_writes[i];
msghdr* hdr = &quic_mhdr_with_cbuf.mhdr()[i].msg_hdr;
CheckIpAndGsoSizeInCbuf(hdr, hdr->msg_control, bw.self_address, 1300);
}
}
TEST_F(QuicLinuxSocketUtilsTest, WriteMultiplePackets_NoPacketsToSend) {
int num_packets_sent;
QuicDeque<BufferedWrite> buffered_writes;
EXPECT_CALL(mock_syscalls_, Sendmmsg(_, _, _, _)).Times(0);
EXPECT_EQ(WriteResult(WRITE_STATUS_ERROR, EINVAL),
TestWriteMultiplePackets(1, buffered_writes.begin(),
buffered_writes.end(), &num_packets_sent));
}
TEST_F(QuicLinuxSocketUtilsTest, WriteMultiplePackets_WriteBlocked) {
int num_packets_sent;
QuicDeque<BufferedWrite> buffered_writes;
buffered_writes.emplace_back(nullptr, 0, QuicIpAddress(),
QuicSocketAddress(QuicIpAddress::Any4(), 0));
EXPECT_CALL(mock_syscalls_, Sendmmsg(_, _, _, _))
.WillOnce(
Invoke([](int fd, mmsghdr* msgvec, unsigned int vlen, int flags) {
errno = EWOULDBLOCK;
return -1;
}));
EXPECT_EQ(WriteResult(WRITE_STATUS_BLOCKED, EWOULDBLOCK),
TestWriteMultiplePackets(1, buffered_writes.begin(),
buffered_writes.end(), &num_packets_sent));
EXPECT_EQ(0, num_packets_sent);
}
TEST_F(QuicLinuxSocketUtilsTest, WriteMultiplePackets_WriteError) {
int num_packets_sent;
QuicDeque<BufferedWrite> buffered_writes;
buffered_writes.emplace_back(nullptr, 0, QuicIpAddress(),
QuicSocketAddress(QuicIpAddress::Any4(), 0));
EXPECT_CALL(mock_syscalls_, Sendmmsg(_, _, _, _))
.WillOnce(
Invoke([](int fd, mmsghdr* msgvec, unsigned int vlen, int flags) {
errno = EPERM;
return -1;
}));
EXPECT_EQ(WriteResult(WRITE_STATUS_ERROR, EPERM),
TestWriteMultiplePackets(1, buffered_writes.begin(),
buffered_writes.end(), &num_packets_sent));
EXPECT_EQ(0, num_packets_sent);
}
TEST_F(QuicLinuxSocketUtilsTest, WriteMultiplePackets_WriteSuccess) {
int num_packets_sent;
QuicDeque<BufferedWrite> buffered_writes;
const int kNumBufferedWrites = 10;
static_assert(kNumBufferedWrites < 256, "Must be less than 256");
std::vector<QuicString> buffer_holder;
for (int i = 0; i < kNumBufferedWrites; ++i) {
size_t buf_len = (i + 1) * 2;
std::ostringstream buffer_ostream;
while (buffer_ostream.str().length() < buf_len) {
buffer_ostream << i;
}
buffer_holder.push_back(buffer_ostream.str().substr(0, buf_len - 1) + '$');
buffered_writes.emplace_back(buffer_holder.back().data(), buf_len,
QuicIpAddress(),
QuicSocketAddress(QuicIpAddress::Any4(), 0));
// Leave the first self_address uninitialized.
if (i != 0) {
ASSERT_TRUE(buffered_writes.back().self_address.FromString("127.0.0.1"));
}
std::ostringstream peer_ip_ostream;
QuicIpAddress peer_ip_address;
peer_ip_ostream << "127.0.1." << i + 1;
ASSERT_TRUE(peer_ip_address.FromString(peer_ip_ostream.str()));
buffered_writes.back().peer_address =
QuicSocketAddress(peer_ip_address, i + 1);
}
InSequence s;
for (int expected_num_packets_sent : {1, 2, 3, 10}) {
SCOPED_TRACE(testing::Message()
<< "expected_num_packets_sent=" << expected_num_packets_sent);
EXPECT_CALL(mock_syscalls_, Sendmmsg(_, _, _, _))
.WillOnce(Invoke([&](int fd, mmsghdr* msgvec, unsigned int vlen,
int flags) {
EXPECT_LE(static_cast<unsigned int>(expected_num_packets_sent), vlen);
for (unsigned int i = 0; i < vlen; ++i) {
const BufferedWrite& buffered_write = buffered_writes[i];
const msghdr& hdr = msgvec[i].msg_hdr;
EXPECT_EQ(1u, hdr.msg_iovlen);
EXPECT_EQ(buffered_write.buffer, hdr.msg_iov->iov_base);
EXPECT_EQ(buffered_write.buf_len, hdr.msg_iov->iov_len);
sockaddr_storage expected_peer_address =
buffered_write.peer_address.generic_address();
EXPECT_EQ(0, memcmp(&expected_peer_address, hdr.msg_name,
sizeof(sockaddr_storage)));
EXPECT_EQ(buffered_write.self_address.IsInitialized(),
hdr.msg_control != nullptr);
}
return expected_num_packets_sent;
}))
.RetiresOnSaturation();
int expected_bytes_written = 0;
for (auto it = buffered_writes.cbegin();
it != buffered_writes.cbegin() + expected_num_packets_sent; ++it) {
expected_bytes_written += it->buf_len;
}
EXPECT_EQ(
WriteResult(WRITE_STATUS_OK, expected_bytes_written),
TestWriteMultiplePackets(1, buffered_writes.cbegin(),
buffered_writes.cend(), &num_packets_sent));
EXPECT_EQ(expected_num_packets_sent, num_packets_sent);
}
}
} // namespace
} // namespace test
} // namespace quic
......@@ -5,7 +5,6 @@
#include "net/third_party/quic/platform/impl/quic_socket_utils.h"
#include <errno.h>
#include <fcntl.h>
#include <linux/net_tstamp.h>
#include <netinet/in.h>
#include <string.h>
......@@ -28,73 +27,6 @@ using std::string;
namespace quic {
QuicMsgHdr::QuicMsgHdr(const char* buffer,
size_t buf_len,
const QuicSocketAddress& peer_address,
char* cbuf,
size_t cbuf_size)
: iov_{const_cast<char*>(buffer), buf_len},
cbuf_(cbuf),
cbuf_size_(cbuf_size),
cmsg_(nullptr) {
// Only support unconnected sockets.
DCHECK(peer_address.IsInitialized());
raw_peer_address_ = peer_address.generic_address();
hdr_.msg_name = &raw_peer_address_;
hdr_.msg_namelen = raw_peer_address_.ss_family == AF_INET
? sizeof(sockaddr_in)
: sizeof(sockaddr_in6);
hdr_.msg_iov = &iov_;
hdr_.msg_iovlen = 1;
hdr_.msg_flags = 0;
hdr_.msg_control = nullptr;
hdr_.msg_controllen = 0;
}
void QuicMsgHdr::SetIpInNextCmsg(const QuicIpAddress& self_address) {
if (!self_address.IsInitialized()) {
return;
}
if (self_address.IsIPv4()) {
QuicSocketUtils::SetIpInfoInCmsgData(
self_address, GetNextCmsgData<in_pktinfo>(IPPROTO_IP, IP_PKTINFO));
} else {
QuicSocketUtils::SetIpInfoInCmsgData(
self_address, GetNextCmsgData<in6_pktinfo>(IPPROTO_IPV6, IPV6_PKTINFO));
}
}
void* QuicMsgHdr::GetNextCmsgDataInternal(int cmsg_level,
int cmsg_type,
size_t data_size) {
// msg_controllen needs to be increased first, otherwise CMSG_NXTHDR will
// return nullptr.
hdr_.msg_controllen += CMSG_SPACE(data_size);
DCHECK_LE(hdr_.msg_controllen, cbuf_size_);
if (cmsg_ == nullptr) {
DCHECK_EQ(nullptr, hdr_.msg_control);
bzero(cbuf_, cbuf_size_);
hdr_.msg_control = cbuf_;
cmsg_ = CMSG_FIRSTHDR(&hdr_);
} else {
DCHECK_NE(nullptr, hdr_.msg_control);
cmsg_ = CMSG_NXTHDR(&hdr_, cmsg_);
}
DCHECK_NE(nullptr, cmsg_) << "Insufficient control buffer space";
cmsg_->cmsg_len = CMSG_LEN(data_size);
cmsg_->cmsg_level = cmsg_level;
cmsg_->cmsg_type = cmsg_type;
return CMSG_DATA(cmsg_);
}
// static
void QuicSocketUtils::GetAddressAndTimestampFromMsghdr(
struct msghdr* hdr,
......@@ -339,38 +271,6 @@ WriteResult QuicSocketUtils::WritePacket(
errno);
}
// static
WriteResult QuicSocketUtils::WritePacket(int fd, const QuicMsgHdr& hdr) {
int rc;
do {
rc = sendmsg(fd, hdr.hdr(), 0);
} while (rc < 0 && errno == EINTR);
if (rc >= 0) {
return WriteResult(WRITE_STATUS_OK, rc);
}
return WriteResult((errno == EAGAIN || errno == EWOULDBLOCK)
? WRITE_STATUS_BLOCKED
: WRITE_STATUS_ERROR,
errno);
}
// static
void QuicSocketUtils::SetIpInfoInCmsgData(const QuicIpAddress& self_address,
void* cmsg_data) {
DCHECK(self_address.IsInitialized());
const QuicString& address_str = self_address.ToPackedString();
if (self_address.IsIPv4()) {
in_pktinfo* pktinfo = static_cast<in_pktinfo*>(cmsg_data);
pktinfo->ipi_ifindex = 0;
memcpy(&pktinfo->ipi_spec_dst, address_str.c_str(), address_str.length());
} else if (self_address.IsIPv6()) {
in6_pktinfo* pktinfo = static_cast<in6_pktinfo*>(cmsg_data);
memcpy(&pktinfo->ipi6_addr, address_str.c_str(), address_str.length());
} else {
QUIC_BUG << "Unrecognized IPAddress";
}
}
// static
int QuicSocketUtils::CreateUDPSocket(const QuicSocketAddress& address,
int32_t receive_buffer_size,
......
......@@ -56,59 +56,8 @@ const int kCmsgSpaceForReadPacket =
kCmsgSpaceForRecvQueueOverflow + kCmsgSpaceForIpv4 + kCmsgSpaceForIpv6 +
kCmsgSpaceForLinuxTimestamping + kCmsgSpaceForTTL;
// QuicMsgHdr is used to build msghdr objects that can be used send packets via
// ::sendmsg.
//
// Example:
// // cbuf holds control messages(cmsgs). The size is determined from what
// // cmsgs will be set for this msghdr.
// char cbuf[kCmsgSpaceForIp + kCmsgSpaceForSegmentSize];
// QuicMsgHdr hdr(packet_buf, packet_buf_len, peer_addr, cbuf, sizeof(cbuf));
//
// // Set IP in cmsgs.
// hdr.SetIpInNextCmsg(self_addr);
//
// // Set GSO size in cmsgs.
// *hdr.GetNextCmsgData<uint16_t>(SOL_UDP, UDP_SEGMENT) = 1200;
//
// QuicSocketUtils::WritePacket(fd, hdr);
class QuicMsgHdr {
public:
QuicMsgHdr(const char* buffer,
size_t buf_len,
const QuicSocketAddress& peer_address,
char* cbuf,
size_t cbuf_size);
// Set IP info in the next cmsg. Both IPv4 and IPv6 are supported.
void SetIpInNextCmsg(const QuicIpAddress& self_address);
template <typename DataType>
DataType* GetNextCmsgData(int cmsg_level, int cmsg_type) {
return reinterpret_cast<DataType*>(
GetNextCmsgDataInternal(cmsg_level, cmsg_type, sizeof(DataType)));
}
const msghdr* hdr() const { return &hdr_; }
protected:
void* GetNextCmsgDataInternal(int cmsg_level,
int cmsg_type,
size_t data_size);
msghdr hdr_;
iovec iov_;
sockaddr_storage raw_peer_address_;
char* cbuf_;
const size_t cbuf_size_;
// The last cmsg populated so far. nullptr means nothing has been populated.
cmsghdr* cmsg_;
};
class QuicSocketUtils {
public:
QuicSocketUtils() = delete;
// Fills in |address| if |hdr| contains IP_PKTINFO or IPV6_PKTINFO. Fills in
// |timestamp| if |hdr| contains |SO_TIMESTAMPING|. |address| and |timestamp|
// must not be null.
......@@ -171,14 +120,6 @@ class QuicSocketUtils {
const QuicIpAddress& self_address,
const QuicSocketAddress& peer_address);
// Writes the packet in |hdr| to the socket, using ::sendmsg.
static WriteResult WritePacket(int fd, const QuicMsgHdr& hdr);
// Set IP(self_address) in |cmsg_data|. Does not touch other fields in the
// containing cmsghdr.
static void SetIpInfoInCmsgData(const QuicIpAddress& self_address,
void* cmsg_data);
// A helper for WritePacket which fills in the cmsg with the supplied self
// address.
// Returns the length of the packet info structure used.
......@@ -192,6 +133,9 @@ class QuicSocketUtils {
int32_t receive_buffer_size,
int32_t send_buffer_size,
bool* overflow_supported);
private:
DISALLOW_COPY_AND_ASSIGN(QuicSocketUtils);
};
} // namespace quic
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment