Commit 8e751655 authored by Brian Sheedy's avatar Brian Sheedy Committed by Commit Bot

Switch WebRTC browser perf tests to histograms

Switches uses of PrintResult in the WebRTC browser perf tests to
PerfResultReporter and whitelists browser_tests for conversion to
histograms before uploading to the perf dashboard.

Bug: 923564
Change-Id: I7ff090bcdeaf3906491b85993456f07313cd4a52
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1856825Reviewed-by: default avatarPatrik Höglund <phoglund@chromium.org>
Reviewed-by: default avatarYuri Wiitala <miu@chromium.org>
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#706699}
parent df8e2714
...@@ -9,7 +9,106 @@ ...@@ -9,7 +9,106 @@
#include "base/strings/stringprintf.h" #include "base/strings/stringprintf.h"
#include "base/values.h" #include "base/values.h"
#include "chrome/test/base/in_process_browser_test.h" #include "chrome/test/base/in_process_browser_test.h"
#include "testing/perf/perf_test.h" #include "testing/perf/perf_result_reporter.h"
namespace {
constexpr char kMetricPrefixAudioReceive[] = "WebRtcAudioReceive.";
constexpr char kMetricPrefixAudioSend[] = "WebRtcAudioSend.";
constexpr char kMetricPrefixVideoSend[] = "WebRtcVideoSend.";
constexpr char kMetricPrefixVideoRecieve[] = "WebRtcVideoReceive.";
constexpr char kMetricPrefixBwe[] = "WebRtcBwe.";
constexpr char kMetricPacketsLostFrames[] = "packets_lost";
constexpr char kMetricGoogJitterRecvMs[] = "goog_jitter_recv";
constexpr char kMetricGoogExpandRatePercent[] = "goog_expand_rate";
constexpr char kMetricGoogSpeechExpandRatePercent[] = "goog_speech_expand_rate";
constexpr char kMetricGoogSecondaryDecodeRatePercent[] =
"goog_secondary_decode_rate";
constexpr char kMetricGoogRttMs[] = "goog_rtt";
constexpr char kMetricPacketsPerSecondPackets[] = "packets_per_second";
constexpr char kMetricGoogFpsSentFps[] = "goog_frame_rate_sent";
constexpr char kMetricGoogFpsInputFps[] = "goog_frame_rate_input";
constexpr char kMetricGoogFirsReceivedUnitless[] = "goog_firs_recv";
constexpr char kMetricGoogNacksReceivedUnitless[] = "goog_nacks_recv";
constexpr char kMetricGoogFrameWidthCount[] = "goog_frame_width";
constexpr char kMetricGoogFrameHeightCount[] = "goog_frame_height";
constexpr char kMetricGoogAvgEncodeMs[] = "goog_avg_encode";
constexpr char kMetricGoogEncodeCpuUsagePercent[] = "goog_encode_cpu_usage";
constexpr char kMetricGoogFpsRecvFps[] = "goog_frame_rate_recv";
constexpr char kMetricGoogFpsOutputFps[] = "goog_frame_rate_output";
constexpr char kMetricGoogActualDelayMs[] = "goog_actual_delay";
constexpr char kMetricGoogTargetDelayMs[] = "goog_target_delay";
constexpr char kMetricGoogDecodeTimeMs[] = "goog_decode_time";
constexpr char kMetricGoogMaxDecodeTimeMs[] = "goog_max_decode_time";
constexpr char kMetricGoogJitterBufferMs[] = "goog_jitter_buffer";
constexpr char kMetricGoogRenderDelayMs[] = "goog_render_delay";
constexpr char kMetricAvailableSendBandwidthBitsPerS[] = "available_send_bw";
constexpr char kMetricAvailableRecvBandwidthBitsPerS[] = "available_recv_bw";
constexpr char kMetricTargetEncodeBitrateBitsPerS[] = "target_encode_bitrate";
constexpr char kMetricActualEncodeBitrateBitsPerS[] = "actual_encode_bitrate";
constexpr char kMetricTransmitBitrateBitsPerS[] = "transmit_bitrate";
perf_test::PerfResultReporter SetUpAudioReceiveReporter(
const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixAudioReceive, story);
reporter.RegisterFyiMetric(kMetricPacketsLostFrames, "frames");
reporter.RegisterFyiMetric(kMetricGoogJitterRecvMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogExpandRatePercent, "%");
reporter.RegisterFyiMetric(kMetricGoogSpeechExpandRatePercent, "%");
reporter.RegisterFyiMetric(kMetricGoogSecondaryDecodeRatePercent, "%");
return reporter;
}
perf_test::PerfResultReporter SetUpAudioSendReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixAudioSend, story);
reporter.RegisterFyiMetric(kMetricGoogJitterRecvMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogRttMs, "ms");
reporter.RegisterFyiMetric(kMetricPacketsPerSecondPackets, "packets");
return reporter;
}
perf_test::PerfResultReporter SetUpVideoSendReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixVideoSend, story);
reporter.RegisterFyiMetric(kMetricGoogFpsSentFps, "fps");
reporter.RegisterFyiMetric(kMetricGoogFpsInputFps, "fps");
reporter.RegisterFyiMetric(kMetricGoogFirsReceivedUnitless, "unitless");
reporter.RegisterFyiMetric(kMetricGoogNacksReceivedUnitless, "unitless");
reporter.RegisterFyiMetric(kMetricGoogFrameWidthCount, "count");
reporter.RegisterFyiMetric(kMetricGoogFrameHeightCount, "count");
reporter.RegisterFyiMetric(kMetricGoogAvgEncodeMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogRttMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogEncodeCpuUsagePercent, "%");
return reporter;
}
perf_test::PerfResultReporter SetUpVideoReceiveReporter(
const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixVideoRecieve, story);
reporter.RegisterFyiMetric(kMetricGoogFpsRecvFps, "fps");
reporter.RegisterFyiMetric(kMetricGoogFpsOutputFps, "fps");
reporter.RegisterFyiMetric(kMetricPacketsLostFrames, "frames");
reporter.RegisterFyiMetric(kMetricGoogFrameWidthCount, "count");
reporter.RegisterFyiMetric(kMetricGoogFrameHeightCount, "count");
reporter.RegisterFyiMetric(kMetricGoogActualDelayMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogTargetDelayMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogDecodeTimeMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogMaxDecodeTimeMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogJitterBufferMs, "ms");
reporter.RegisterFyiMetric(kMetricGoogRenderDelayMs, "ms");
return reporter;
}
perf_test::PerfResultReporter SetUpBweReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixBwe, story);
reporter.RegisterFyiMetric(kMetricAvailableSendBandwidthBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricAvailableRecvBandwidthBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricTargetEncodeBitrateBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricActualEncodeBitrateBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricTransmitBitrateBitsPerS, "bits/s");
return reporter;
}
} // namespace
static std::string Statistic(const std::string& statistic, static std::string Statistic(const std::string& statistic,
const std::string& bucket) { const std::string& bucket) {
...@@ -23,59 +122,53 @@ static std::string Statistic(const std::string& statistic, ...@@ -23,59 +122,53 @@ static std::string Statistic(const std::string& statistic,
} }
static void MaybePrintResultsForAudioReceive( static void MaybePrintResultsForAudioReceive(
const std::string& ssrc, const base::DictionaryValue& pc_dict, const std::string& ssrc,
const std::string& modifier) { const base::DictionaryValue& pc_dict,
const std::string& story) {
std::string value; std::string value;
if (!pc_dict.GetString(Statistic("audioOutputLevel", ssrc), &value)) { if (!pc_dict.GetString(Statistic("audioOutputLevel", ssrc), &value)) {
// Not an audio receive stream. // Not an audio receive stream.
return; return;
} }
auto reporter = SetUpAudioReceiveReporter(story);
EXPECT_TRUE(pc_dict.GetString(Statistic("packetsLost", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("packetsLost", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricPacketsLostFrames, value);
"audio_misc", modifier, "packets_lost", value, "frames", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googJitterReceived", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googJitterReceived", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogJitterRecvMs, value);
"audio_rx", modifier, "goog_jitter_recv", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googExpandRate", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googExpandRate", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogExpandRatePercent, value);
"audio_rates", modifier, "goog_expand_rate", value, "%", false);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("googSpeechExpandRate", ssrc), &value)); pc_dict.GetString(Statistic("googSpeechExpandRate", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogSpeechExpandRatePercent, value);
"audio_rates", modifier, "goog_speech_expand_rate", value, "%", false);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("googSecondaryDecodedRate", ssrc), &value)); pc_dict.GetString(Statistic("googSecondaryDecodedRate", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogSecondaryDecodeRatePercent, value);
"audio_rates", modifier, "goog_secondary_decoded_rate", value, "%",
false);
} }
static void MaybePrintResultsForAudioSend( static void MaybePrintResultsForAudioSend(const std::string& ssrc,
const std::string& ssrc, const base::DictionaryValue& pc_dict, const base::DictionaryValue& pc_dict,
const std::string& modifier) { const std::string& story) {
std::string value; std::string value;
if (!pc_dict.GetString(Statistic("audioInputLevel", ssrc), &value)) { if (!pc_dict.GetString(Statistic("audioInputLevel", ssrc), &value)) {
// Not an audio send stream. // Not an audio send stream.
return; return;
} }
auto reporter = SetUpAudioSendReporter(story);
EXPECT_TRUE(pc_dict.GetString(Statistic("googJitterReceived", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googJitterReceived", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogJitterRecvMs, value);
"audio_tx", modifier, "goog_jitter_recv", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googRtt", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googRtt", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogRttMs, value);
"audio_tx", modifier, "goog_rtt", value, "ms", false);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("packetsSentPerSecond", ssrc), &value)); pc_dict.GetString(Statistic("packetsSentPerSecond", ssrc), &value));
perf_test::PrintResult("audio_tx", modifier, "packets_sent_per_second", value, reporter.AddResult(kMetricPacketsPerSecondPackets, value);
"packets", false);
} }
static void MaybePrintResultsForVideoSend( static void MaybePrintResultsForVideoSend(const std::string& ssrc,
const std::string& ssrc, const base::DictionaryValue& pc_dict, const base::DictionaryValue& pc_dict,
const std::string& modifier) { const std::string& story) {
std::string value; std::string value;
if (!pc_dict.GetString(Statistic("googFrameRateSent", ssrc), &value)) { if (!pc_dict.GetString(Statistic("googFrameRateSent", ssrc), &value)) {
// Not a video send stream. // Not a video send stream.
...@@ -85,86 +178,70 @@ static void MaybePrintResultsForVideoSend( ...@@ -85,86 +178,70 @@ static void MaybePrintResultsForVideoSend(
// Graph these by unit: the dashboard expects all stats in one graph to have // Graph these by unit: the dashboard expects all stats in one graph to have
// the same unit (e.g. ms, fps, etc). Most graphs, like video_fps, will also // the same unit (e.g. ms, fps, etc). Most graphs, like video_fps, will also
// be populated by the counterparts on the video receiving side. // be populated by the counterparts on the video receiving side.
perf_test::PrintResult( auto reporter = SetUpVideoSendReporter(story);
"video_fps", modifier, "goog_frame_rate_sent", value, "fps", false); reporter.AddResult(kMetricGoogFpsSentFps, value);
EXPECT_TRUE(pc_dict.GetString(Statistic("googFrameRateInput", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googFrameRateInput", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogFpsInputFps, value);
"video_fps", modifier, "goog_frame_rate_input", value, "fps", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googFirsReceived", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googFirsReceived", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogFirsReceivedUnitless, value);
"video_misc", modifier, "goog_firs_recv", value, "", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googNacksReceived", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googNacksReceived", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogNacksReceivedUnitless, value);
"video_misc", modifier, "goog_nacks_recv", value, "", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googFrameWidthSent", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googFrameWidthSent", ssrc), &value));
perf_test::PrintResult("video_resolution", modifier, "goog_frame_width_sent", reporter.AddResult(kMetricGoogFrameWidthCount, value);
value, "pixels", false);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("googFrameHeightSent", ssrc), &value)); pc_dict.GetString(Statistic("googFrameHeightSent", ssrc), &value));
perf_test::PrintResult("video_resolution", modifier, "goog_frame_height_sent", reporter.AddResult(kMetricGoogFrameHeightCount, value);
value, "pixels", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googAvgEncodeMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googAvgEncodeMs", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogAvgEncodeMs, value);
"video_tx", modifier, "goog_avg_encode_ms", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googRtt", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googRtt", ssrc), &value));
perf_test::PrintResult("video_tx", modifier, "goog_rtt", value, "ms", false); reporter.AddResult(kMetricGoogRttMs, value);
EXPECT_TRUE(pc_dict.GetString( EXPECT_TRUE(pc_dict.GetString(
Statistic("googEncodeUsagePercent", ssrc), &value)); Statistic("googEncodeUsagePercent", ssrc), &value));
perf_test::PrintResult("video_cpu_usage", modifier, reporter.AddResult(kMetricGoogEncodeCpuUsagePercent, value);
"goog_encode_usage_percent", value, "%", false);
} }
static void MaybePrintResultsForVideoReceive( static void MaybePrintResultsForVideoReceive(
const std::string& ssrc, const base::DictionaryValue& pc_dict, const std::string& ssrc,
const std::string& modifier) { const base::DictionaryValue& pc_dict,
const std::string& story) {
std::string value; std::string value;
if (!pc_dict.GetString(Statistic("googFrameRateReceived", ssrc), &value)) { if (!pc_dict.GetString(Statistic("googFrameRateReceived", ssrc), &value)) {
// Not a video receive stream. // Not a video receive stream.
return; return;
} }
perf_test::PrintResult( auto reporter = SetUpVideoReceiveReporter(story);
"video_fps", modifier, "goog_frame_rate_recv", value, "fps", false); reporter.AddResult(kMetricGoogFpsRecvFps, value);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("googFrameRateOutput", ssrc), &value)); pc_dict.GetString(Statistic("googFrameRateOutput", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogFpsOutputFps, value);
"video_fps", modifier, "goog_frame_rate_output", value, "fps", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("packetsLost", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("packetsLost", ssrc), &value));
perf_test::PrintResult("video_misc", modifier, "packets_lost", value, reporter.AddResult(kMetricPacketsLostFrames, value);
"frames", false);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("googFrameWidthReceived", ssrc), &value)); pc_dict.GetString(Statistic("googFrameWidthReceived", ssrc), &value));
perf_test::PrintResult("video_resolution", modifier, "goog_frame_width_recv", reporter.AddResult(kMetricGoogFrameWidthCount, value);
value, "pixels", false);
EXPECT_TRUE( EXPECT_TRUE(
pc_dict.GetString(Statistic("googFrameHeightReceived", ssrc), &value)); pc_dict.GetString(Statistic("googFrameHeightReceived", ssrc), &value));
perf_test::PrintResult("video_resolution", modifier, "goog_frame_height_recv", reporter.AddResult(kMetricGoogFrameHeightCount, value);
value, "pixels", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googCurrentDelayMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googCurrentDelayMs", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogActualDelayMs, value);
"video_rx", modifier, "goog_current_delay_ms", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googTargetDelayMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googTargetDelayMs", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogTargetDelayMs, value);
"video_rx", modifier, "goog_target_delay_ms", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googDecodeMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googDecodeMs", ssrc), &value));
perf_test::PrintResult("video_rx", modifier, "goog_decode_ms", value, "ms", reporter.AddResult(kMetricGoogDecodeTimeMs, value);
false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googMaxDecodeMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googMaxDecodeMs", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogMaxDecodeTimeMs, value);
"video_rx", modifier, "goog_max_decode_ms", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googJitterBufferMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googJitterBufferMs", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogJitterBufferMs, value);
"video_rx", modifier, "goog_jitter_buffer_ms", value, "ms", false);
EXPECT_TRUE(pc_dict.GetString(Statistic("googRenderDelayMs", ssrc), &value)); EXPECT_TRUE(pc_dict.GetString(Statistic("googRenderDelayMs", ssrc), &value));
perf_test::PrintResult( reporter.AddResult(kMetricGoogRenderDelayMs, value);
"video_rx", modifier, "goog_render_delay_ms", value, "ms", false);
} }
static std::string ExtractSsrcIdentifier(const std::string& key) { static std::string ExtractSsrcIdentifier(const std::string& key) {
...@@ -195,30 +272,26 @@ namespace test { ...@@ -195,30 +272,26 @@ namespace test {
void PrintBweForVideoMetrics(const base::DictionaryValue& pc_dict, void PrintBweForVideoMetrics(const base::DictionaryValue& pc_dict,
const std::string& modifier, const std::string& modifier,
const std::string& video_codec) { const std::string& video_codec) {
std::string video_modifier = std::string story = video_codec.empty() ? "baseline_story" : video_codec;
video_codec.empty() ? modifier : modifier + "_" + video_codec; story += modifier;
const std::string kBweStatsKey = "bweforvideo"; const std::string kBweStatsKey = "bweforvideo";
std::string value; std::string value;
auto reporter = SetUpBweReporter(story);
ASSERT_TRUE(pc_dict.GetString( ASSERT_TRUE(pc_dict.GetString(
Statistic("googAvailableSendBandwidth", kBweStatsKey), &value)); Statistic("googAvailableSendBandwidth", kBweStatsKey), &value));
perf_test::PrintResult("bwe_stats", video_modifier, "available_send_bw", reporter.AddResult(kMetricAvailableSendBandwidthBitsPerS, value);
value, "bit/s", false);
ASSERT_TRUE(pc_dict.GetString( ASSERT_TRUE(pc_dict.GetString(
Statistic("googAvailableReceiveBandwidth", kBweStatsKey), &value)); Statistic("googAvailableReceiveBandwidth", kBweStatsKey), &value));
perf_test::PrintResult("bwe_stats", video_modifier, "available_recv_bw", reporter.AddResult(kMetricAvailableRecvBandwidthBitsPerS, value);
value, "bit/s", false);
ASSERT_TRUE(pc_dict.GetString( ASSERT_TRUE(pc_dict.GetString(
Statistic("googTargetEncBitrate", kBweStatsKey), &value)); Statistic("googTargetEncBitrate", kBweStatsKey), &value));
perf_test::PrintResult("bwe_stats", video_modifier, "target_enc_bitrate", reporter.AddResult(kMetricTargetEncodeBitrateBitsPerS, value);
value, "bit/s", false);
ASSERT_TRUE(pc_dict.GetString( ASSERT_TRUE(pc_dict.GetString(
Statistic("googActualEncBitrate", kBweStatsKey), &value)); Statistic("googActualEncBitrate", kBweStatsKey), &value));
perf_test::PrintResult("bwe_stats", video_modifier, "actual_enc_bitrate", reporter.AddResult(kMetricActualEncodeBitrateBitsPerS, value);
value, "bit/s", false);
ASSERT_TRUE(pc_dict.GetString( ASSERT_TRUE(pc_dict.GetString(
Statistic("googTransmitBitrate", kBweStatsKey), &value)); Statistic("googTransmitBitrate", kBweStatsKey), &value));
perf_test::PrintResult("bwe_stats", video_modifier, "transmit_bitrate", value, reporter.AddResult(kMetricTransmitBitrateBitsPerS, value);
"bit/s", false);
} }
void PrintMetricsForAllStreams(const base::DictionaryValue& pc_dict, void PrintMetricsForAllStreams(const base::DictionaryValue& pc_dict,
...@@ -231,8 +304,8 @@ void PrintMetricsForAllStreams(const base::DictionaryValue& pc_dict, ...@@ -231,8 +304,8 @@ void PrintMetricsForAllStreams(const base::DictionaryValue& pc_dict,
void PrintMetricsForSendStreams(const base::DictionaryValue& pc_dict, void PrintMetricsForSendStreams(const base::DictionaryValue& pc_dict,
const std::string& modifier, const std::string& modifier,
const std::string& video_codec) { const std::string& video_codec) {
std::string video_modifier = std::string story = video_codec.empty() ? "baseline_story" : video_codec;
video_codec.empty() ? modifier : modifier + "_" + video_codec; story += modifier;
const base::DictionaryValue* stats_dict; const base::DictionaryValue* stats_dict;
ASSERT_TRUE(pc_dict.GetDictionary("stats", &stats_dict)); ASSERT_TRUE(pc_dict.GetDictionary("stats", &stats_dict));
std::set<std::string> ssrc_identifiers = FindAllSsrcIdentifiers(*stats_dict); std::set<std::string> ssrc_identifiers = FindAllSsrcIdentifiers(*stats_dict);
...@@ -240,16 +313,16 @@ void PrintMetricsForSendStreams(const base::DictionaryValue& pc_dict, ...@@ -240,16 +313,16 @@ void PrintMetricsForSendStreams(const base::DictionaryValue& pc_dict,
auto ssrc_iterator = ssrc_identifiers.begin(); auto ssrc_iterator = ssrc_identifiers.begin();
for (; ssrc_iterator != ssrc_identifiers.end(); ++ssrc_iterator) { for (; ssrc_iterator != ssrc_identifiers.end(); ++ssrc_iterator) {
const std::string& ssrc = *ssrc_iterator; const std::string& ssrc = *ssrc_iterator;
MaybePrintResultsForAudioSend(ssrc, pc_dict, modifier); MaybePrintResultsForAudioSend(ssrc, pc_dict, story);
MaybePrintResultsForVideoSend(ssrc, pc_dict, video_modifier); MaybePrintResultsForVideoSend(ssrc, pc_dict, story);
} }
} }
void PrintMetricsForRecvStreams(const base::DictionaryValue& pc_dict, void PrintMetricsForRecvStreams(const base::DictionaryValue& pc_dict,
const std::string& modifier, const std::string& modifier,
const std::string& video_codec) { const std::string& video_codec) {
std::string video_modifier = std::string story = video_codec.empty() ? "baseline_story" : video_codec;
video_codec.empty() ? modifier : modifier + "_" + video_codec; story += modifier;
const base::DictionaryValue* stats_dict; const base::DictionaryValue* stats_dict;
ASSERT_TRUE(pc_dict.GetDictionary("stats", &stats_dict)); ASSERT_TRUE(pc_dict.GetDictionary("stats", &stats_dict));
std::set<std::string> ssrc_identifiers = FindAllSsrcIdentifiers(*stats_dict); std::set<std::string> ssrc_identifiers = FindAllSsrcIdentifiers(*stats_dict);
...@@ -257,8 +330,8 @@ void PrintMetricsForRecvStreams(const base::DictionaryValue& pc_dict, ...@@ -257,8 +330,8 @@ void PrintMetricsForRecvStreams(const base::DictionaryValue& pc_dict,
auto ssrc_iterator = ssrc_identifiers.begin(); auto ssrc_iterator = ssrc_identifiers.begin();
for (; ssrc_iterator != ssrc_identifiers.end(); ++ssrc_iterator) { for (; ssrc_iterator != ssrc_identifiers.end(); ++ssrc_iterator) {
const std::string& ssrc = *ssrc_iterator; const std::string& ssrc = *ssrc_iterator;
MaybePrintResultsForAudioReceive(ssrc, pc_dict, modifier); MaybePrintResultsForAudioReceive(ssrc, pc_dict, story);
MaybePrintResultsForVideoReceive(ssrc, pc_dict, video_modifier); MaybePrintResultsForVideoReceive(ssrc, pc_dict, story);
} }
} }
......
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
#include "chrome/browser/media/webrtc/webrtc_browsertest_common.h" #include "chrome/browser/media/webrtc/webrtc_browsertest_common.h"
#include "content/public/common/content_switches.h" #include "content/public/common/content_switches.h"
#include "media/base/media_switches.h" #include "media/base/media_switches.h"
#include "testing/perf/perf_test.h" #include "testing/perf/perf_result_reporter.h"
#include "third_party/blink/public/common/features.h" #include "third_party/blink/public/common/features.h"
namespace content { namespace content {
...@@ -26,6 +26,35 @@ const char kMainWebrtcTestHtmlPage[] = "/webrtc/webrtc_jsep01_test.html"; ...@@ -26,6 +26,35 @@ const char kMainWebrtcTestHtmlPage[] = "/webrtc/webrtc_jsep01_test.html";
const char kInboundRtp[] = "inbound-rtp"; const char kInboundRtp[] = "inbound-rtp";
const char kOutboundRtp[] = "outbound-rtp"; const char kOutboundRtp[] = "outbound-rtp";
constexpr int kBitsPerByte = 8;
constexpr char kMetricPrefixAudioStats[] = "WebRtcAudioStats.";
constexpr char kMetricPrefixVideoStats[] = "WebRtcVideoStats.";
constexpr char kMetricPrefixGetStats[] = "WebRtcGetStats.";
constexpr char kMetricSendRateBitsPerS[] = "send_rate";
constexpr char kMetricReceiveRateBitsPerS[] = "receive_rate";
constexpr char kMetricInvocationTimeMs[] = "invocation_time";
perf_test::PerfResultReporter SetUpAudioReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixAudioStats, story);
reporter.RegisterFyiMetric(kMetricSendRateBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricReceiveRateBitsPerS, "bits/s");
return reporter;
}
perf_test::PerfResultReporter SetUpVideoReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixVideoStats, story);
reporter.RegisterFyiMetric(kMetricSendRateBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricReceiveRateBitsPerS, "bits/s");
return reporter;
}
perf_test::PerfResultReporter SetUpGetStatsReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixGetStats, story);
reporter.RegisterFyiMetric(kMetricInvocationTimeMs, "ms");
return reporter;
}
enum class GetStatsVariation { enum class GetStatsVariation {
PROMISE_BASED, PROMISE_BASED,
CALLBACK_BASED CALLBACK_BASED
...@@ -200,13 +229,11 @@ class WebRtcStatsPerfBrowserTest : public WebRtcTestBase { ...@@ -200,13 +229,11 @@ class WebRtcStatsPerfBrowserTest : public WebRtcTestBase {
(audio_bytes_received_after - audio_bytes_received_before) / (audio_bytes_received_after - audio_bytes_received_before) /
measure_duration_seconds; measure_duration_seconds;
std::string audio_codec_modifier = "_" + audio_codec; auto reporter = SetUpAudioReporter(audio_codec);
perf_test::PrintResult( reporter.AddResult(kMetricSendRateBitsPerS,
"audio", audio_codec_modifier, "send_rate", audio_send_rate, audio_send_rate * kBitsPerByte);
"bytes/second", false); reporter.AddResult(kMetricReceiveRateBitsPerS,
perf_test::PrintResult( audio_receive_rate * kBitsPerByte);
"audio", audio_codec_modifier, "receive_rate", audio_receive_rate,
"bytes/second", false);
} }
if (video_codec != kUseDefaultVideoCodec) { if (video_codec != kUseDefaultVideoCodec) {
double video_bytes_sent_after = GetVideoBytesSent(report.get()); double video_bytes_sent_after = GetVideoBytesSent(report.get());
...@@ -219,15 +246,14 @@ class WebRtcStatsPerfBrowserTest : public WebRtcTestBase { ...@@ -219,15 +246,14 @@ class WebRtcStatsPerfBrowserTest : public WebRtcTestBase {
(video_bytes_received_after - video_bytes_received_before) / (video_bytes_received_after - video_bytes_received_before) /
measure_duration_seconds; measure_duration_seconds;
std::string video_codec_modifier = std::string story =
"_" + (video_codec_print_modifier.empty() (video_codec_print_modifier.empty() ? video_codec
? video_codec
: video_codec_print_modifier); : video_codec_print_modifier);
perf_test::PrintResult("video", video_codec_modifier, "send_rate", auto reporter = SetUpVideoReporter(story);
video_send_rate, "bytes/second", false); reporter.AddResult(kMetricSendRateBitsPerS,
perf_test::PrintResult( video_send_rate * kBitsPerByte);
"video", video_codec_modifier, "receive_rate", video_receive_rate, reporter.AddResult(kMetricReceiveRateBitsPerS,
"bytes/second", false); video_receive_rate * kBitsPerByte);
} }
EndCall(); EndCall();
...@@ -252,14 +278,9 @@ class WebRtcStatsPerfBrowserTest : public WebRtcTestBase { ...@@ -252,14 +278,9 @@ class WebRtcStatsPerfBrowserTest : public WebRtcTestBase {
MeasureGetStatsCallbackPerformance(right_tab_)) / 2.0; MeasureGetStatsCallbackPerformance(right_tab_)) / 2.0;
break; break;
} }
perf_test::PrintResult( auto reporter = SetUpGetStatsReporter(
"getStats", variation == GetStatsVariation::PROMISE_BASED ? "promise" : "callback");
(variation == GetStatsVariation::PROMISE_BASED) ? reporter.AddResult(kMetricInvocationTimeMs, invocation_time);
"_promise" : "_callback",
"invocation_time",
invocation_time,
"milliseconds",
false);
EndCall(); EndCall();
} }
......
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
#include "content/public/test/browser_test_utils.h" #include "content/public/test/browser_test_utils.h"
#include "media/base/media_switches.h" #include "media/base/media_switches.h"
#include "net/test/embedded_test_server/embedded_test_server.h" #include "net/test/embedded_test_server/embedded_test_server.h"
#include "testing/perf/perf_test.h" #include "testing/perf/perf_result_reporter.h"
#include "third_party/blink/public/common/features.h" #include "third_party/blink/public/common/features.h"
#include "ui/gl/gl_switches.h" #include "ui/gl/gl_switches.h"
...@@ -49,10 +49,42 @@ static const char kVsyncEventName[] = "Display::DrawAndSwap"; ...@@ -49,10 +49,42 @@ static const char kVsyncEventName[] = "Display::DrawAndSwap";
static const char kVideoFrameSubmitterEventName[] = "VideoFrameSubmitter"; static const char kVideoFrameSubmitterEventName[] = "VideoFrameSubmitter";
static const char kEventMatchKey[] = "Timestamp"; static const char kEventMatchKey[] = "Timestamp";
static const char kTestResultString[] = "TestVideoDisplayPerf";
static const char kMainWebrtcTestHtmlPage[] = static const char kMainWebrtcTestHtmlPage[] =
"/webrtc/webrtc_video_display_perf_test.html"; "/webrtc/webrtc_video_display_perf_test.html";
constexpr char kMetricPrefixVideoDisplayPerf[] = "WebRtcVideoDisplayPerf.";
constexpr char kMetricSkippedFramesPercent[] = "skipped_frames";
constexpr char kMetricPassingToRenderAlgoLatencyUs[] =
"passing_to_render_algorithm_latency";
constexpr char kMetricRenderAlgoLatencyUs[] = "render_algorithm_latency";
constexpr char kMetricCompositorPickingFrameLatencyUs[] =
"compositor_picking_frame_latency";
constexpr char kMetricCompositorResourcePreparationLatencyUs[] =
"compositor_resource_preparation_latency";
constexpr char kMetricVsyncLatencyUs[] = "vsync_latency";
constexpr char kMetricTotalControlledLatencyUs[] = "total_controlled_latency";
constexpr char kMetricTotalLatencyUs[] = "total_latency";
constexpr char kMetricPostDecodeToRasterLatencyUs[] =
"post_decode_to_raster_latency";
constexpr char kMetricWebRtcDecodeLatencyUs[] = "webrtc_decode_latency";
perf_test::PerfResultReporter SetUpReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixVideoDisplayPerf, story);
reporter.RegisterImportantMetric(kMetricSkippedFramesPercent, "percent");
reporter.RegisterImportantMetric(kMetricPassingToRenderAlgoLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricRenderAlgoLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricCompositorPickingFrameLatencyUs,
"us");
reporter.RegisterImportantMetric(
kMetricCompositorResourcePreparationLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricVsyncLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricTotalControlledLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricTotalLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricPostDecodeToRasterLatencyUs, "us");
reporter.RegisterImportantMetric(kMetricWebRtcDecodeLatencyUs, "us");
return reporter;
}
struct VideoDisplayPerfTestConfig { struct VideoDisplayPerfTestConfig {
int width; int width;
int height; int height;
...@@ -60,36 +92,13 @@ struct VideoDisplayPerfTestConfig { ...@@ -60,36 +92,13 @@ struct VideoDisplayPerfTestConfig {
bool disable_render_smoothness_algorithm; bool disable_render_smoothness_algorithm;
}; };
void CalculateMeanAndMax(const std::vector<double>& inputs, std::string VectorToString(const std::vector<double>& values) {
double* mean, std::string ret = "";
double* std_dev, for (double val : values) {
double* max) { ret += base::StringPrintf("%.0lf,", val);
double sum = 0.0;
double sqr_sum = 0.0;
double max_so_far = 0.0;
size_t count = inputs.size();
for (const auto& input : inputs) {
sum += input;
sqr_sum += input * input;
max_so_far = std::max(input, max_so_far);
} }
*max = max_so_far; // Strip of trailing comma.
*mean = sum / count; return ret.substr(0, ret.length() - 1);
*std_dev = sqrt(std::max(0.0, count * sqr_sum - sum * sum)) / count;
}
void PrintMeanAndMax(const std::string& var_name,
const std::string& name_modifier,
const std::vector<double>& vars) {
double mean = 0.0;
double std_dev = 0.0;
double max = 0.0;
CalculateMeanAndMax(vars, &mean, &std_dev, &max);
perf_test::PrintResultMeanAndError(
kTestResultString, name_modifier, var_name + " Mean",
base::StringPrintf("%.0lf,%.0lf", mean, std_dev), "μs", true);
perf_test::PrintResult(kTestResultString, name_modifier, var_name + " Max",
base::StringPrintf("%.0lf", max), "μs", true);
} }
void FindEvents(trace_analyzer::TraceAnalyzer* analyzer, void FindEvents(trace_analyzer::TraceAnalyzer* analyzer,
...@@ -402,35 +411,36 @@ class WebRtcVideoDisplayPerfBrowserTest ...@@ -402,35 +411,36 @@ class WebRtcVideoDisplayPerfBrowserTest
std::string smoothness_indicator = std::string smoothness_indicator =
test_config_.disable_render_smoothness_algorithm ? "_DisableSmoothness" test_config_.disable_render_smoothness_algorithm ? "_DisableSmoothness"
: ""; : "";
std::string name_modifier = base::StringPrintf( std::string story = base::StringPrintf(
"%s_%dp%df%s", video_codec.c_str(), test_config_.height, "%s_%dp%df%s", video_codec.c_str(), test_config_.height,
test_config_.fps, smoothness_indicator.c_str()); test_config_.fps, smoothness_indicator.c_str());
perf_test::PrintResult( auto reporter = SetUpReporter(story);
kTestResultString, name_modifier, "Skipped frames", reporter.AddResult(kMetricSkippedFramesPercent,
base::StringPrintf("%.2lf", skipped_frame_percentage_), "percent", base::StringPrintf("%.2lf", skipped_frame_percentage_));
true);
// We identify intervals in a way that can help us easily bisect the source // We identify intervals in a way that can help us easily bisect the source
// of added latency in case of a regression. From these intervals, "Render // of added latency in case of a regression. From these intervals, "Render
// Algorithm" can take random amount of times based on the vsync cycle it is // Algorithm" can take random amount of times based on the vsync cycle it is
// closest to. Therefore, "Total Controlled Latency" refers to the total // closest to. Therefore, "Total Controlled Latency" refers to the total
// times without that section for semi-consistent results. // times without that section for semi-consistent results.
PrintMeanAndMax("Passing to Render Algorithm Latency", name_modifier, reporter.AddResultList(kMetricPassingToRenderAlgoLatencyUs,
enqueue_frame_durations_); VectorToString(enqueue_frame_durations_));
PrintMeanAndMax("Render Algorithm Latency", name_modifier, reporter.AddResultList(kMetricRenderAlgoLatencyUs,
set_frame_durations_); VectorToString(set_frame_durations_));
PrintMeanAndMax("Compositor Picking Frame Latency", name_modifier, reporter.AddResultList(kMetricCompositorPickingFrameLatencyUs,
get_frame_durations_); VectorToString(get_frame_durations_));
PrintMeanAndMax("Compositor Resource Preparation Latency", name_modifier, reporter.AddResultList(kMetricCompositorResourcePreparationLatencyUs,
resource_ready_durations_); VectorToString(resource_ready_durations_));
PrintMeanAndMax("Vsync Latency", name_modifier, vsync_durations_); reporter.AddResultList(kMetricVsyncLatencyUs,
PrintMeanAndMax("Total Controlled Latency", name_modifier, VectorToString(vsync_durations_));
total_controlled_durations_); reporter.AddResultList(kMetricTotalControlledLatencyUs,
PrintMeanAndMax("Total Latency", name_modifier, total_durations_); VectorToString(total_controlled_durations_));
reporter.AddResultList(kMetricTotalLatencyUs,
PrintMeanAndMax("Post-decode-to-raster latency", name_modifier, VectorToString(total_durations_));
video_frame_submmitter_latencies_);
PrintMeanAndMax("WebRTC decode latency", name_modifier, reporter.AddResultList(kMetricPostDecodeToRasterLatencyUs,
webrtc_decode_latencies_); VectorToString(video_frame_submmitter_latencies_));
reporter.AddResultList(kMetricWebRtcDecodeLatencyUs,
VectorToString(webrtc_decode_latencies_));
} }
VideoDisplayPerfTestConfig test_config_; VideoDisplayPerfTestConfig test_config_;
......
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
#include "content/public/browser/web_contents.h" #include "content/public/browser/web_contents.h"
#include "content/public/common/content_switches.h" #include "content/public/common/content_switches.h"
#include "media/base/media_switches.h" #include "media/base/media_switches.h"
#include "testing/perf/perf_test.h" #include "testing/perf/perf_result_reporter.h"
#include "ui/gl/gl_switches.h" #include "ui/gl/gl_switches.h"
namespace { namespace {
...@@ -22,6 +22,19 @@ static const char kMainWebrtcTestHtmlPage[] = ...@@ -22,6 +22,19 @@ static const char kMainWebrtcTestHtmlPage[] =
static const char kInboundRtp[] = "inbound-rtp"; static const char kInboundRtp[] = "inbound-rtp";
static const char kOutboundRtp[] = "outbound-rtp"; static const char kOutboundRtp[] = "outbound-rtp";
constexpr int kBitsPerByte = 8;
constexpr char kMetricPrefixHighBitrate[] = "WebRtcHighBitrateVideo.";
constexpr char kMetricSendRateBitsPerS[] = "send_rate";
constexpr char kMetricReceiveRateBitsPerS[] = "receive_rate";
perf_test::PerfResultReporter SetUpReporter(const std::string& story) {
perf_test::PerfResultReporter reporter(kMetricPrefixHighBitrate, story);
reporter.RegisterFyiMetric(kMetricSendRateBitsPerS, "bits/s");
reporter.RegisterFyiMetric(kMetricReceiveRateBitsPerS, "bits/s");
return reporter;
}
// Sums up "RTC[In/Out]boundRTPStreamStats.bytes_[received/sent]" values. // Sums up "RTC[In/Out]boundRTPStreamStats.bytes_[received/sent]" values.
double GetTotalRTPStreamBytes(content::TestStatsReportDictionary* report, double GetTotalRTPStreamBytes(content::TestStatsReportDictionary* report,
const char* type, const char* type,
...@@ -138,10 +151,10 @@ IN_PROC_BROWSER_TEST_F(WebRtcVideoHighBitrateBrowserTest, ...@@ -138,10 +151,10 @@ IN_PROC_BROWSER_TEST_F(WebRtcVideoHighBitrateBrowserTest,
(video_bytes_received_after - video_bytes_received_before) / (video_bytes_received_after - video_bytes_received_before) /
duration_in_seconds; duration_in_seconds;
perf_test::PrintResult("video", "", "send_rate", video_send_rate, auto reporter = SetUpReporter("baseline_story");
"bytes/second", false); reporter.AddResult(kMetricSendRateBitsPerS, video_send_rate * kBitsPerByte);
perf_test::PrintResult("video", "", "receive_rate", video_receive_rate, reporter.AddResult(kMetricReceiveRateBitsPerS,
"bytes/second", false); video_receive_rate * kBitsPerByte);
HangUp(left_tab); HangUp(left_tab);
HangUp(right_tab); HangUp(right_tab);
......
...@@ -56,6 +56,7 @@ DATA_FORMAT_UNKNOWN = 'unknown' ...@@ -56,6 +56,7 @@ DATA_FORMAT_UNKNOWN = 'unknown'
# are okay with potentially encountering issues. # are okay with potentially encountering issues.
GTEST_CONVERSION_WHITELIST = [ GTEST_CONVERSION_WHITELIST = [
'angle_perftests', 'angle_perftests',
'browser_tests',
'cc_perftests', 'cc_perftests',
'components_perftests', 'components_perftests',
'gpu_perftests', 'gpu_perftests',
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment