Introduce a QUIC trace format.

Merge internal change: 194843554

R=rch@chromium.org

Change-Id: Ia7c236db9045f3ea59df088b0f18d3717dbc32cc
Reviewed-on: https://chromium-review.googlesource.com/1038023Reviewed-by: Ryan Hamilton <rch@chromium.org>
Commit-Queue: Victor Vasiliev <vasilvv@chromium.org>
Cr-Commit-Position: refs/heads/master@{#555628}

net/BUILD.gn

@@ -1271,6 +1271,8 @@ component("net") {
       "quic/core/quic_tag.h",
       "quic/core/quic_time.cc",
       "quic/core/quic_time.h",
+      "quic/core/quic_trace_visitor.cc",
+      "quic/core/quic_trace_visitor.h",
       "quic/core/quic_transmission_info.cc",
       "quic/core/quic_transmission_info.h",
       "quic/core/quic_types.cc",
@@ -2279,6 +2281,7 @@ proto_library("net_quic_proto") {
 
   sources = [
     "quic/core/proto/cached_network_parameters.proto",
+    "quic/core/proto/quic_trace.proto",
     "quic/core/proto/source_address_token.proto",
   ]
   cc_generator_options = "dllexport_decl=NET_EXPORT_PRIVATE:"
@@ -5190,6 +5193,7 @@ test("net_unittests") {
     "quic/core/quic_sustained_bandwidth_recorder_test.cc",
     "quic/core/quic_tag_test.cc",
     "quic/core/quic_time_test.cc",
+    "quic/core/quic_trace_visitor_test.cc",
     "quic/core/quic_unacked_packet_map_test.cc",
     "quic/core/quic_utils_test.cc",
     "quic/core/quic_version_manager_test.cc",

net/quic/core/proto/quic_trace.proto

+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file describes a format for a trace containing all events related to the
+// QUIC connection.  It is primarily focused on representing information related
+// to congestion control and loss recovery.  It's entirely up to implementation
+// which fields to populate; the consumers of the data should not assume all of
+// the frames are recorded in either direction.
+//
+// This format is intended to be usable by any QUIC implementation.  It is
+// primarily based on IETF QUIC (draft 11), but has a few legacy gQUIC entries,
+// and can be potentially extended in the future.
+syntax = "proto2";
+
+option optimize_for = LITE_RUNTIME;
+
+package quic_trace;
+
+enum FrameType {
+  UNKNOWN_FRAME = 0;
+
+  STREAM = 1;
+  ACK = 2;
+  RESET_STREAM = 3;
+  CONNECTION_CLOSE = 4;
+  MAX_DATA = 5;
+  MAX_STREAM_DATA = 6;
+  PING = 7;
+  BLOCKED = 8;
+  STREAM_BLOCKED = 9;
+  PADDING = 10;
+};
+
+// Metadata for STREAM frames.
+message StreamFrameInfo {
+  optional uint64 stream_id = 1;
+  optional bool fin = 2;
+  optional uint64 length = 3;
+  optional uint64 offset = 4;
+};
+
+// The intervals are closed, i.e. the interval represented here is
+// [first_packet, last_packet].
+message AckBlock {
+  optional uint64 first_packet = 1;
+  optional uint64 last_packet = 2;
+};
+
+// Metadata for ACK frames.
+message AckInfo {
+  repeated AckBlock acked_packets = 1;
+  optional uint64 ack_delay_us = 2;
+};
+
+// Metadata for RST_STREAM frames.
+message ResetStreamInfo {
+  optional uint64 stream_id = 1;
+  optional uint32 application_error_code = 2;
+  optional uint64 final_offset = 3;
+};
+
+// Metadata for CONNECTION_CLOSE/APPLICATION_CLOSE frames.
+message CloseInfo {
+  optional uint32 error_code = 1;
+  optional string reason_phrase = 2;
+};
+
+// Metadata for MAX_DATA/MAX_STREAM_DATA frames.
+message FlowControlInfo {
+  optional uint64 max_data = 1;
+  optional uint64 stream_id = 2;
+};
+
+// A message representing a frame, either sent or received.
+message Frame {
+  optional FrameType frame_type = 1;
+
+  optional StreamFrameInfo stream_frame_info = 2;
+  optional AckInfo ack_info = 3;
+  optional ResetStreamInfo reset_stream_info = 4;
+  optional CloseInfo close_info = 5;
+  optional FlowControlInfo flow_control_info = 6;
+};
+
+// Metadata that represents transport stack's understanding of the current state
+// of the transport channel.
+message TransportState {
+  optional uint64 min_rtt_us = 1;
+  // Smoothed RTT, usually computed using EWMA.
+  optional uint64 smoothed_rtt_us = 2;
+  // The latest RTT measureent available.
+  optional uint64 last_rtt_us = 3;
+
+  optional uint64 in_flight_bytes = 4;
+  optional uint64 cwnd_bytes = 5;
+  // Pacing rate, in bits per second.
+  optional uint64 pacing_rate_bps = 6;
+
+  // Any arbitrary information about congestion control state that is not
+  // representable via parameters above.
+  optional string congestion_control_state = 7;
+};
+
+enum EncryptionLevel {
+  ENCRYPTION_UNKNOWN = 0;
+
+  ENCRYPTION_INITIAL = 1;
+  ENCRYPTION_0RTT = 2;
+  ENCRYPTION_1RTT = 3;
+};
+
+enum EventType {
+  UNKNOWN_EVENT = 0;
+
+  PACKET_SENT = 1;
+  PACKET_RECEIVED = 2;
+  PACKET_LOST = 3;
+};
+
+// An event that has occurred over duration of the connection.
+message Event {
+  optional uint64 time_us = 1;
+  optional EventType event_type = 2;
+
+  optional uint64 packet_number = 3;
+  repeated Frame frames = 4;
+  optional uint64 packet_size = 5;
+  optional EncryptionLevel encryption_level = 6;
+  // State of the transport stack after the event has happened.
+  optional TransportState transport_state = 7;
+};
+
+message Trace {
+  // QUIC version tag, as represented on wire.  Should be always 4 bytes long.
+  optional bytes protocol_version = 1;
+
+  // Source and destination connection ID.  If multiple connection IDs are used,
+  // record the first one used with short-form header.
+  optional bytes source_connection_id = 2;
+  optional bytes destination_connection_id = 3;
+
+  repeated Event events = 4;
+};

net/quic/core/quic_trace_visitor.h

+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef NET_QUIC_CORE_QUIC_TRACE_VISITOR_H_
+#define NET_QUIC_CORE_QUIC_TRACE_VISITOR_H_
+
+#include "net/quic/core/proto/quic_trace.pb.h"
+#include "net/quic/core/quic_connection.h"
+#include "net/quic/core/quic_types.h"
+
+namespace net {
+
+// Records a QUIC trace protocol buffer for a QuicConnection.  It's the
+// responsibility of the user of this visitor to process or store the resulting
+// trace, which can be accessed via trace().
+class QUIC_EXPORT_PRIVATE QuicTraceVisitor : public QuicConnectionDebugVisitor {
+ public:
+  explicit QuicTraceVisitor(const QuicConnection* connection);
+
+  void OnPacketSent(const SerializedPacket& serialized_packet,
+                    QuicPacketNumber original_packet_number,
+                    TransmissionType transmission_type,
+                    QuicTime sent_time) override;
+
+  void OnIncomingAck(const QuicAckFrame& ack_frame,
+                     QuicTime ack_receive_time,
+                     QuicPacketNumber largest_observed,
+                     bool rtt_updated,
+                     QuicPacketNumber least_unacked_sent_packet) override;
+
+  void OnPacketLoss(QuicPacketNumber lost_packet_number,
+                    TransmissionType transmission_type,
+                    QuicTime detection_time) override;
+
+  void OnWindowUpdateFrame(const QuicWindowUpdateFrame& frame,
+                           const QuicTime& receive_time) override;
+
+  void OnSuccessfulVersionNegotiation(
+      const ParsedQuicVersion& version) override;
+
+  // Returns a mutable pointer to the trace.  The trace is owned by the
+  // visitor, but can be moved using Swap() method after the connection is
+  // finished.
+  quic_trace::Trace* trace() { return &trace_; }
+
+ private:
+  // Converts QuicTime into a microsecond delta w.r.t. the beginning of the
+  // connection.
+  uint64_t ConvertTimestampToRecordedFormat(QuicTime timestamp);
+  // Populates a quic_trace::Frame message from |frame|.
+  void PopulateFrameInfo(const QuicFrame& frame,
+                         quic_trace::Frame* frame_record);
+  // Populates a quic_trace::TransportState message from the associated
+  // connection.
+  void PopulateTransportState(quic_trace::TransportState* state);
+
+  quic_trace::Trace trace_;
+  const QuicConnection* connection_;
+  const QuicTime start_time_;
+};
+
+};  // namespace net
+
+#endif  // NET_QUIC_CORE_QUIC_TRACE_VISITOR_H_

net/quic/core/quic_trace_visitor_test.cc

+// Copyright (c) 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "net/quic/core/quic_trace_visitor.h"
+
+#include "net/quic/core/quic_constants.h"
+#include "net/quic/platform/api/quic_test.h"
+#include "net/quic/test_tools/simulator/quic_endpoint.h"
+#include "net/quic/test_tools/simulator/simulator.h"
+#include "net/quic/test_tools/simulator/switch.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace net {
+namespace {
+
+using std::string;
+
+const QuicByteCount kTransferSize = 1000 * kMaxPacketSize;
+const QuicByteCount kTestStreamNumber = 3;
+const QuicTime::Delta kDelay = QuicTime::Delta::FromMilliseconds(20);
+
+// The trace for this test is generated using a simulator transfer.
+class QuicTraceVisitorTest : public QuicTest {
+ public:
+  QuicTraceVisitorTest() {
+    simulator::Simulator simulator;
+    simulator::QuicEndpoint client(&simulator, "Client", "Server",
+                                   Perspective::IS_CLIENT, 42);
+    simulator::QuicEndpoint server(&simulator, "Server", "Client",
+                                   Perspective::IS_SERVER, 42);
+
+    const QuicBandwidth kBandwidth = QuicBandwidth::FromKBitsPerSecond(1000);
+    const QuicByteCount kBdp = kBandwidth * (2 * kDelay);
+
+    // Create parameters such that some loss is observed.
+    simulator::Switch network_switch(&simulator, "Switch", 8, 0.5 * kBdp);
+    simulator::SymmetricLink client_link(&client, network_switch.port(1),
+                                         2 * kBandwidth, kDelay);
+    simulator::SymmetricLink server_link(&server, network_switch.port(2),
+                                         kBandwidth, kDelay);
+
+    QuicTraceVisitor visitor(client.connection());
+    client.connection()->set_debug_visitor(&visitor);
+
+    // Transfer about a megabyte worth of data from client to server.
+    const QuicTime::Delta kDeadline =
+        3 * kBandwidth.TransferTime(kTransferSize);
+    client.AddBytesToTransfer(kTransferSize);
+    bool simulator_result = simulator.RunUntilOrTimeout(
+        [&]() { return server.bytes_received() >= kTransferSize; }, kDeadline);
+    CHECK(simulator_result);
+
+    // Save the trace and ensure some loss was observed.
+    trace_.Swap(visitor.trace());
+    CHECK_NE(0u, client.connection()->GetStats().packets_retransmitted);
+    packets_sent_ = client.connection()->GetStats().packets_sent;
+  }
+
+  std::vector<quic_trace::Event> AllEventsWithType(
+      quic_trace::EventType event_type) {
+    std::vector<quic_trace::Event> result;
+    for (const auto& event : trace_.events()) {
+      if (event.event_type() == event_type) {
+        result.push_back(event);
+      }
+    }
+    return result;
+  }
+
+ protected:
+  quic_trace::Trace trace_;
+  QuicPacketCount packets_sent_;
+};
+
+TEST_F(QuicTraceVisitorTest, ConnectionId) {
+  char expected_cid[] = {0, 0, 0, 0, 0, 0, 0, 42};
+  EXPECT_EQ(string(expected_cid, sizeof(expected_cid)),
+            trace_.destination_connection_id());
+}
+
+TEST_F(QuicTraceVisitorTest, Version) {
+  string version = trace_.protocol_version();
+  ASSERT_EQ(4u, version.size());
+  EXPECT_EQ('Q', version[0]);
+}
+
+// Check that basic metadata about sent packets is recorded.
+TEST_F(QuicTraceVisitorTest, SentPacket) {
+  auto sent_packets = AllEventsWithType(quic_trace::PACKET_SENT);
+  EXPECT_EQ(packets_sent_, sent_packets.size());
+  ASSERT_GT(sent_packets.size(), 0u);
+
+  EXPECT_EQ(sent_packets[0].packet_size(), kDefaultMaxPacketSize);
+  EXPECT_EQ(sent_packets[0].packet_number(), 1u);
+}
+
+// Ensure that every stream frame that was sent is recorded.
+TEST_F(QuicTraceVisitorTest, SentStream) {
+  auto sent_packets = AllEventsWithType(quic_trace::PACKET_SENT);
+
+  QuicIntervalSet<QuicStreamOffset> offsets;
+  for (const quic_trace::Event& packet : sent_packets) {
+    for (const quic_trace::Frame& frame : packet.frames()) {
+      if (frame.frame_type() != quic_trace::STREAM) {
+        continue;
+      }
+
+      const quic_trace::StreamFrameInfo& info = frame.stream_frame_info();
+      if (info.stream_id() != kTestStreamNumber) {
+        continue;
+      }
+
+      ASSERT_GT(info.length(), 0u);
+      offsets.Add(QuicIntervalSet<QuicStreamOffset>(
+          info.offset(), info.offset() + info.length()));
+    }
+  }
+
+  ASSERT_EQ(1u, offsets.Size());
+  EXPECT_EQ(0u, offsets.begin()->min());
+  EXPECT_EQ(kTransferSize, offsets.rbegin()->max());
+}
+
+// Ensure that all packets are either acknowledged or lost.
+TEST_F(QuicTraceVisitorTest, AckPackets) {
+  QuicIntervalSet<QuicPacketNumber> packets;
+  for (const quic_trace::Event& packet : trace_.events()) {
+    if (packet.event_type() == quic_trace::PACKET_RECEIVED) {
+      for (const quic_trace::Frame& frame : packet.frames()) {
+        if (frame.frame_type() != quic_trace::ACK) {
+          continue;
+        }
+
+        const quic_trace::AckInfo& info = frame.ack_info();
+        for (const auto& block : info.acked_packets()) {
+          packets.Add(block.first_packet(), block.last_packet() + 1);
+        }
+      }
+    }
+    if (packet.event_type() == quic_trace::PACKET_LOST) {
+      packets.Add(packet.packet_number(), packet.packet_number() + 1);
+    }
+  }
+
+  ASSERT_EQ(1u, packets.Size());
+  EXPECT_EQ(1u, packets.begin()->min());
+  // We leave some room (20 packets) for the packets which did not receive
+  // conclusive status at the end of simulation.
+  EXPECT_GT(packets.rbegin()->max(), packets_sent_ - 20);
+}
+
+TEST_F(QuicTraceVisitorTest, TransportState) {
+  auto acks = AllEventsWithType(quic_trace::PACKET_RECEIVED);
+  ASSERT_EQ(1, acks[0].frames_size());
+  ASSERT_EQ(quic_trace::ACK, acks[0].frames(0).frame_type());
+
+  // Check that min-RTT at the end is a reasonable approximation.
+  EXPECT_LE((4 * kDelay).ToMicroseconds() * 1.,
+            acks.rbegin()->transport_state().min_rtt_us());
+  EXPECT_GE((4 * kDelay).ToMicroseconds() * 1.25,
+            acks.rbegin()->transport_state().min_rtt_us());
+}
+
+}  // namespace
+}  // namespace net