This splits the main browser test so the webrtc-internals perf test is

the only one to run with a reference file and moves the normal tests to
use the generating fake device (i.e. the spinning green ball). This is
fine since the main test doesn't care, whereas the perf test must have
a somewhat realistic video to give realistic encode/decode numbers.

The CPU test can be thrown away since we have telemetry tests for that
now.

BUG=375240

Review URL: https://codereview.chromium.org/294053002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@272828 0039d316-1c4b-4281-b951-d872f2087c98

chrome/browser/media/chrome_webrtc_audio_quality_browsertest.cc

@@ -10,7 +10,6 @@
 #include "base/process/launch.h"
 #include "base/scoped_native_library.h"
 #include "base/strings/stringprintf.h"
-#include "base/win/windows_version.h"
 #include "chrome/browser/media/webrtc_browsertest_base.h"
 #include "chrome/browser/media/webrtc_browsertest_common.h"
 #include "chrome/browser/profiles/profile.h"
@@ -350,13 +349,10 @@ INSTANTIATE_TEST_CASE_P(WebRtcAudioQualityBrowserTests,
 
 IN_PROC_BROWSER_TEST_P(WebRtcAudioQualityBrowserTest,
                        MAYBE_MANUAL_TestAudioQuality) {
-#if defined(OS_WIN)
-  if (base::win::GetVersion() < base::win::VERSION_VISTA) {
-    // It would take work to implement this on XP; not prioritized right now.
+  if (OnWinXp()) {
     LOG(ERROR) << "This test is not implemented for Windows XP.";
     return;
   }
-#endif
   ASSERT_TRUE(test::HasReferenceFilesInCheckout());
   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 

chrome/browser/media/chrome_webrtc_browsertest.cc

@@ -3,50 +3,28 @@
 // found in the LICENSE file.
 
 #include "base/command_line.h"
-#include "base/file_util.h"
-#include "base/json/json_reader.h"
-#include "base/memory/scoped_ptr.h"
-#include "base/path_service.h"
-#include "base/process/launch.h"
-#include "base/process/process_metrics.h"
-#include "base/strings/string_split.h"
 #include "base/strings/stringprintf.h"
-#include "base/synchronization/waitable_event.h"
-#include "base/test/test_timeouts.h"
-#include "base/time/time.h"
-#include "chrome/browser/browser_process.h"
 #include "chrome/browser/media/webrtc_browsertest_base.h"
 #include "chrome/browser/media/webrtc_browsertest_common.h"
-#include "chrome/browser/media/webrtc_browsertest_perf.h"
 #include "chrome/browser/ui/browser.h"
 #include "chrome/browser/ui/browser_tabstrip.h"
 #include "chrome/browser/ui/tabs/tab_strip_model.h"
 #include "chrome/common/chrome_switches.h"
 #include "chrome/test/base/in_process_browser_test.h"
 #include "chrome/test/base/ui_test_utils.h"
-#include "content/public/browser/notification_service.h"
-#include "content/public/browser/render_process_host.h"
 #include "content/public/test/browser_test_utils.h"
 #include "media/base/media_switches.h"
 #include "net/test/embedded_test_server/embedded_test_server.h"
-#include "testing/perf/perf_test.h"
-
-// For fine-grained suppression.
-#if defined(OS_WIN)
-#include "base/win/windows_version.h"
-#endif
 
 static const char kMainWebrtcTestHtmlPage[] =
     "/webrtc/webrtc_jsep01_test.html";
 
-// Top-level integration test for WebRTC. The test methods here must run
-// sequentially since they use a server binary on the system (hence they are
-// tagged as MANUAL). In addition, they need the reference videos which require
-// the webrtc.DEPS solution, which is not generally available on Chromium bots.
+// Top-level integration test for WebRTC. It always uses fake devices; see
+// WebRtcWebcamBrowserTest for a test that acquires any real webcam on the
+// system.
 class WebRtcBrowserTest : public WebRtcTestBase,
                           public testing::WithParamInterface<bool> {
  public:
-  WebRtcBrowserTest() {}
   virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
     DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
   }
@@ -55,78 +33,11 @@ class WebRtcBrowserTest : public WebRtcTestBase,
     // Ensure the infobar is enabled, since we expect that in this test.
     EXPECT_FALSE(command_line->HasSwitch(switches::kUseFakeUIForMediaStream));
 
-    // Play a suitable, somewhat realistic video file.
-    base::FilePath input_video = test::GetReferenceFilesDir()
-        .Append(test::kReferenceFileName360p)
-        .AddExtension(test::kY4mFileExtension);
-    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
-                                   input_video);
+    // Always use fake devices.
     command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
 
     // Flag used by TestWebAudioMediaStream to force garbage collection.
     command_line->AppendSwitchASCII(switches::kJavaScriptFlags, "--expose-gc");
-
-    bool enable_audio_track_processing = GetParam();
-    if (enable_audio_track_processing)
-      command_line->AppendSwitch(switches::kEnableAudioTrackProcessing);
-  }
-
-  void PrintProcessMetrics(base::ProcessMetrics* process_metrics,
-                           const std::string& suffix) {
-    perf_test::PrintResult("cpu", "", "cpu" + suffix,
-                           process_metrics->GetCPUUsage(),
-                           "%", true);
-    perf_test::PrintResult("memory", "", "ws_peak" + suffix,
-                           process_metrics->GetPeakWorkingSetSize(),
-                           "bytes", true);
-    perf_test::PrintResult("memory", "", "ws_final" + suffix,
-                           process_metrics->GetWorkingSetSize(),
-                           "bytes", false);
-
-    size_t private_mem;
-    size_t shared_mem;
-    if (process_metrics->GetMemoryBytes(&private_mem, &shared_mem)) {
-      perf_test::PrintResult("memory", "", "private_mem_final" + suffix,
-                             private_mem, "bytes", false);
-      perf_test::PrintResult("memory", "", "shared_mem_final" + suffix,
-                             shared_mem, "bytes", false);
-    }
-  }
-
-  // Tries to extract data from peerConnectionDataStore in the webrtc-internals
-  // tab. The caller owns the parsed data. Returns NULL on failure.
-  base::DictionaryValue* GetWebrtcInternalsData(
-      content::WebContents* webrtc_internals_tab) {
-    std::string all_stats_json = ExecuteJavascript(
-        "window.domAutomationController.send("
-        "    JSON.stringify(peerConnectionDataStore));",
-        webrtc_internals_tab);
-
-    base::Value* parsed_json = base::JSONReader::Read(all_stats_json);
-    base::DictionaryValue* result;
-    if (parsed_json && parsed_json->GetAsDictionary(&result))
-      return result;
-
-    return NULL;
-  }
-
-  const base::DictionaryValue* GetDataOnFirstPeerConnection(
-      const base::DictionaryValue* all_data) {
-    base::DictionaryValue::Iterator iterator(*all_data);
-
-    const base::DictionaryValue* result;
-    if (!iterator.IsAtEnd() && iterator.value().GetAsDictionary(&result))
-      return result;
-
-    return NULL;
-  }
-
-  bool OnWinXp() {
-#if defined(OS_WIN)
-    return base::win::GetVersion() <= base::win::VERSION_XP;
-#else
-    return false;
-#endif
   }
 };
 
@@ -139,7 +50,6 @@ IN_PROC_BROWSER_TEST_P(WebRtcBrowserTest,
                        MANUAL_RunsAudioVideoWebRTCCallInTwoTabs) {
   if (OnWinXp()) return;
 
-  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 
   content::WebContents* left_tab =
@@ -159,115 +69,14 @@ IN_PROC_BROWSER_TEST_P(WebRtcBrowserTest,
   WaitForVideoToPlay(right_tab);
 
   HangUp(left_tab);
-}
-
-IN_PROC_BROWSER_TEST_P(WebRtcBrowserTest, MANUAL_CpuUsage15Seconds) {
-  if (OnWinXp()) return;
-
-  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
-  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
-
-  base::FilePath results_file;
-  ASSERT_TRUE(base::CreateTemporaryFile(&results_file));
-
-  content::WebContents* left_tab =
-      OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
-
-#if defined(OS_MACOSX)
-  // Don't measure renderer CPU on mac: requires a mach broker we don't have
-  // access to from the browser test.
-  scoped_ptr<base::ProcessMetrics> browser_process_metrics(
-      base::ProcessMetrics::CreateProcessMetrics(
-          base::Process::Current().handle(), NULL));
-  browser_process_metrics->GetCPUUsage();
-#else
-  // Measure rendering CPU on platforms that support it.
-  base::ProcessHandle renderer_pid =
-      left_tab->GetRenderProcessHost()->GetHandle();
-  scoped_ptr<base::ProcessMetrics> renderer_process_metrics(
-      base::ProcessMetrics::CreateProcessMetrics(renderer_pid));
-  renderer_process_metrics->GetCPUUsage();
-
-  scoped_ptr<base::ProcessMetrics> browser_process_metrics(
-      base::ProcessMetrics::CreateProcessMetrics(
-          base::Process::Current().handle()));
-  browser_process_metrics->GetCPUUsage();
-#endif
-
-  content::WebContents* right_tab =
-      OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
-
-  SetupPeerconnectionWithLocalStream(left_tab);
-  SetupPeerconnectionWithLocalStream(right_tab);
-
-  NegotiateCall(left_tab, right_tab);
-
-  test::SleepInJavascript(left_tab, 15000);
-
-  HangUp(left_tab);
-
-#if !defined(OS_MACOSX)
-  PrintProcessMetrics(renderer_process_metrics.get(), "_r");
-#endif
-  PrintProcessMetrics(browser_process_metrics.get(), "_b");
-}
-
-// This is manual for its long execution time.
-IN_PROC_BROWSER_TEST_P(WebRtcBrowserTest,
-                       MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetrics) {
-  if (OnWinXp()) return;
-
-  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
-  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
-
-  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100) <<
-      "This is a long-running test; you must specify "
-      "--ui-test-action-max-timeout to have a value of at least 100000.";
-
-  content::WebContents* left_tab =
-      OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
-  content::WebContents* right_tab =
-      OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
-
-  SetupPeerconnectionWithLocalStream(left_tab);
-  SetupPeerconnectionWithLocalStream(right_tab);
-
-  NegotiateCall(left_tab, right_tab);
-
-  StartDetectingVideo(left_tab, "remote-view");
-  StartDetectingVideo(right_tab, "remote-view");
-
-  WaitForVideoToPlay(left_tab);
-  WaitForVideoToPlay(right_tab);
-
-  // Let values stabilize, bandwidth ramp up, etc.
-  test::SleepInJavascript(left_tab, 60000);
-
-  // Start measurements.
-  chrome::AddTabAt(browser(), GURL(), -1, true);
-  ui_test_utils::NavigateToURL(browser(), GURL("chrome://webrtc-internals"));
-  content::WebContents* webrtc_internals_tab =
-      browser()->tab_strip_model()->GetActiveWebContents();
-
-  test::SleepInJavascript(left_tab, 10000);
-
-  scoped_ptr<base::DictionaryValue> all_data(
-      GetWebrtcInternalsData(webrtc_internals_tab));
-  ASSERT_TRUE(all_data.get() != NULL);
-
-  const base::DictionaryValue* first_pc_dict =
-      GetDataOnFirstPeerConnection(all_data.get());
-  ASSERT_TRUE(first_pc_dict != NULL);
-  test::PrintBweForVideoMetrics(*first_pc_dict);
-  test::PrintMetricsForAllStreams(*first_pc_dict);
-
-  HangUp(left_tab);
+  HangUp(right_tab);
 }
 
 IN_PROC_BROWSER_TEST_P(WebRtcBrowserTest, MANUAL_TestWebAudioMediaStream) {
+  // This tests against crash regressions for the WebAudio-MediaStream
+  // integration.
   if (OnWinXp()) return;
 
-  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
   GURL url(embedded_test_server()->GetURL("/webrtc/webaudio_crash.html"));
   ui_test_utils::NavigateToURL(browser(), url);

chrome/browser/media/chrome_webrtc_disable_encryption_flag_browsertest.cc

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 #include "base/command_line.h"
-#include "base/win/windows_version.h"
 #include "chrome/browser/media/webrtc_browsertest_base.h"
 #include "chrome/browser/media/webrtc_browsertest_common.h"
 #include "chrome/common/chrome_version_info.h"
@@ -51,11 +50,8 @@ class WebRtcDisableEncryptionFlagBrowserTest : public WebRtcTestBase {
 // Makes a call and checks that there's encryption or not in the SDP offer.
 IN_PROC_BROWSER_TEST_F(WebRtcDisableEncryptionFlagBrowserTest,
                        VerifyEncryption) {
-// Flaky timeout on a webrtc Win XP bot. http://crbug.com/368163.
-#if defined (OS_WIN)
-  if (base::win::GetVersion() < base::win::VERSION_VISTA)
-    return;
-#endif
+  if (!OnWinXp())
+    return;  // Flaky timeout on a webrtc Win XP bot. http://crbug.com/368163.
 
   ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
 
@@ -94,4 +90,5 @@ IN_PROC_BROWSER_TEST_F(WebRtcDisableEncryptionFlagBrowserTest,
             ExecuteJavascript("hasSeenCryptoInSdp()", left_tab));
 
   HangUp(left_tab);
+  HangUp(right_tab);
 }

chrome/browser/media/chrome_webrtc_perf_browsertest.cc

+// Copyright 2014 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/command_line.h"
+#include "base/file_util.h"
+#include "base/json/json_reader.h"
+#include "base/memory/scoped_ptr.h"
+#include "base/strings/string_split.h"
+#include "base/strings/stringprintf.h"
+#include "base/test/test_timeouts.h"
+#include "base/time/time.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/media/webrtc_browsertest_base.h"
+#include "chrome/browser/media/webrtc_browsertest_common.h"
+#include "chrome/browser/media/webrtc_browsertest_perf.h"
+#include "chrome/browser/ui/browser.h"
+#include "chrome/browser/ui/browser_tabstrip.h"
+#include "chrome/browser/ui/tabs/tab_strip_model.h"
+#include "chrome/common/chrome_switches.h"
+#include "chrome/test/base/in_process_browser_test.h"
+#include "chrome/test/base/ui_test_utils.h"
+#include "content/public/test/browser_test_utils.h"
+#include "media/base/media_switches.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
+#include "testing/perf/perf_test.h"
+
+static const char kMainWebrtcTestHtmlPage[] =
+    "/webrtc/webrtc_jsep01_test.html";
+
+// Performance browsertest for WebRTC. This test is manual since it takes long
+// to execute and requires the reference files provided by the webrtc.DEPS
+// solution (which is only available on WebRTC internal bots).
+class WebRtcPerfBrowserTest : public WebRtcTestBase {
+ public:
+  virtual void SetUpInProcessBrowserTestFixture() OVERRIDE {
+    DetectErrorsInJavaScript();  // Look for errors in our rather complex js.
+  }
+
+  virtual void SetUpCommandLine(CommandLine* command_line) OVERRIDE {
+    // Ensure the infobar is enabled, since we expect that in this test.
+    EXPECT_FALSE(command_line->HasSwitch(switches::kUseFakeUIForMediaStream));
+
+    // Play a suitable, somewhat realistic video file.
+    base::FilePath input_video = test::GetReferenceFilesDir()
+        .Append(test::kReferenceFileName360p)
+        .AddExtension(test::kY4mFileExtension);
+    command_line->AppendSwitchPath(switches::kUseFileForFakeVideoCapture,
+                                   input_video);
+    command_line->AppendSwitch(switches::kUseFakeDeviceForMediaStream);
+  }
+
+  // Tries to extract data from peerConnectionDataStore in the webrtc-internals
+  // tab. The caller owns the parsed data. Returns NULL on failure.
+  base::DictionaryValue* GetWebrtcInternalsData(
+      content::WebContents* webrtc_internals_tab) {
+    std::string all_stats_json = ExecuteJavascript(
+        "window.domAutomationController.send("
+        "    JSON.stringify(peerConnectionDataStore));",
+        webrtc_internals_tab);
+
+    base::Value* parsed_json = base::JSONReader::Read(all_stats_json);
+    base::DictionaryValue* result;
+    if (parsed_json && parsed_json->GetAsDictionary(&result))
+      return result;
+
+    return NULL;
+  }
+
+  const base::DictionaryValue* GetDataOnFirstPeerConnection(
+      const base::DictionaryValue* all_data) {
+    base::DictionaryValue::Iterator iterator(*all_data);
+
+    const base::DictionaryValue* result;
+    if (!iterator.IsAtEnd() && iterator.value().GetAsDictionary(&result))
+      return result;
+
+    return NULL;
+  }
+};
+
+// This is manual for its long execution time.
+IN_PROC_BROWSER_TEST_F(WebRtcPerfBrowserTest,
+                       MANUAL_RunsAudioVideoCall60SecsAndLogsInternalMetrics) {
+  if (OnWinXp()) return;
+
+  ASSERT_TRUE(test::HasReferenceFilesInCheckout());
+  ASSERT_TRUE(embedded_test_server()->InitializeAndWaitUntilReady());
+
+  ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 100) <<
+      "This is a long-running test; you must specify "
+      "--ui-test-action-max-timeout to have a value of at least 100000.";
+
+  content::WebContents* left_tab =
+      OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
+  content::WebContents* right_tab =
+      OpenTestPageAndGetUserMediaInNewTab(kMainWebrtcTestHtmlPage);
+
+  SetupPeerconnectionWithLocalStream(left_tab);
+  SetupPeerconnectionWithLocalStream(right_tab);
+
+  NegotiateCall(left_tab, right_tab);
+
+  StartDetectingVideo(left_tab, "remote-view");
+  StartDetectingVideo(right_tab, "remote-view");
+
+  WaitForVideoToPlay(left_tab);
+  WaitForVideoToPlay(right_tab);
+
+  // Let values stabilize, bandwidth ramp up, etc.
+  test::SleepInJavascript(left_tab, 60000);
+
+  // Start measurements.
+  chrome::AddTabAt(browser(), GURL(), -1, true);
+  ui_test_utils::NavigateToURL(browser(), GURL("chrome://webrtc-internals"));
+  content::WebContents* webrtc_internals_tab =
+      browser()->tab_strip_model()->GetActiveWebContents();
+
+  test::SleepInJavascript(left_tab, 10000);
+
+  scoped_ptr<base::DictionaryValue> all_data(
+      GetWebrtcInternalsData(webrtc_internals_tab));
+  ASSERT_TRUE(all_data.get() != NULL);
+
+  const base::DictionaryValue* first_pc_dict =
+      GetDataOnFirstPeerConnection(all_data.get());
+  ASSERT_TRUE(first_pc_dict != NULL);
+  test::PrintBweForVideoMetrics(*first_pc_dict);
+  test::PrintMetricsForAllStreams(*first_pc_dict);
+
+  HangUp(left_tab);
+  HangUp(right_tab);
+}

chrome/browser/media/chrome_webrtc_typing_detection_browsertest.cc

@@ -134,4 +134,5 @@ IN_PROC_BROWSER_TEST_F(WebRtcTypingDetectionBrowserTest,
   test::SleepInJavascript(left_tab, 10000);
 
   HangUp(left_tab);
+  HangUp(right_tab);
 }

chrome/browser/media/chrome_webrtc_video_quality_browsertest.cc

@@ -32,11 +32,6 @@
 #include "testing/perf/perf_test.h"
 #include "ui/gl/gl_switches.h"
 
-// For fine-grained suppression on flaky tests.
-#if defined(OS_WIN)
-#include "base/win/windows_version.h"
-#endif
-
 static const base::FilePath::CharType kFrameAnalyzerExecutable[] =
 #if defined(OS_WIN)
     FILE_PATH_LITERAL("frame_analyzer.exe");
@@ -320,12 +315,8 @@ INSTANTIATE_TEST_CASE_P(
 
 IN_PROC_BROWSER_TEST_P(WebRtcVideoQualityBrowserTest,
                        MANUAL_TestVideoQuality) {
-
-#if defined(OS_WIN)
-  // Fails on XP. http://crbug.com/353078
-  if (base::win::GetVersion() <= base::win::VERSION_XP)
-    return;
-#endif
+  if (OnWinXp())
+    return;  // Fails on XP. http://crbug.com/353078.
 
   ASSERT_GE(TestTimeouts::action_max_timeout().InSeconds(), 150) <<
       "This is a long-running test; you must specify "

chrome/browser/media/webrtc_browsertest_base.cc

@@ -20,6 +20,11 @@
 #include "content/public/test/browser_test_utils.h"
 #include "net/test/embedded_test_server/embedded_test_server.h"
 
+#if defined(OS_WIN)
+// For fine-grained suppression.
+#include "base/win/windows_version.h"
+#endif
+
 const char WebRtcTestBase::kAudioVideoCallConstraints[] =
     "'{audio: true, video: true}'";
 const char WebRtcTestBase::kAudioVideoCallConstraintsQVGA[] =
@@ -347,3 +352,11 @@ bool WebRtcTestBase::HasWebcamAvailableOnSystem(
       ExecuteJavascript("HasVideoSourceOnSystem();", tab_contents);
   return result == "has-video-source";
 }
+
+bool WebRtcTestBase::OnWinXp() const {
+#if defined(OS_WIN)
+  return base::win::GetVersion() <= base::win::VERSION_XP;
+#else
+  return false;
+#endif
+}

chrome/browser/media/webrtc_browsertest_base.h

@@ -110,6 +110,9 @@ class WebRtcTestBase : public InProcessBrowserTest {
   // Methods to check what devices we have on the system.
   bool HasWebcamAvailableOnSystem(content::WebContents* tab_contents) const;
 
+  // Returns true if we're on WinXP, that lovely operating system of bliss.
+  bool OnWinXp() const;
+
  private:
   void CloseInfoBarInTab(content::WebContents* tab_contents,
                          infobars::InfoBar* infobar) const;

chrome/chrome_tests.gypi

@@ -1254,6 +1254,7 @@
         'browser/media/chrome_webrtc_browsertest.cc',
         'browser/media/chrome_webrtc_disable_encryption_flag_browsertest.cc',
         'browser/media/chrome_webrtc_audio_quality_browsertest.cc',
+        'browser/media/chrome_webrtc_perf_browsertest.cc',
         'browser/media/chrome_webrtc_typing_detection_browsertest.cc',
         'browser/media/chrome_webrtc_video_quality_browsertest.cc',
         'browser/media/chrome_webrtc_webcam_browsertest.cc',