[ChromeDriver] Add replay end-to-end tests to run_all_tests.py

This change adds the end-to-end tests for the client replay component
of ChromeDriver to the ChromeDriver waterfall by incorporating them
into the existing run_all_tests.py. This will extend the waterfall
coverage to include the ChromeDriver log-replay component.

Bug: chromedriver:2501
Change-Id: I6e35675d2598a2a33c67340e0ee73e05201c3439
Reviewed-on: https://chromium-review.googlesource.com/1174894
Commit-Queue: Cole Winstanley <cwinstanley@google.com>
Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Reviewed-by: John Chen <johnchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#584518}

chrome/test/chromedriver/log_replay/client_replay_test.py

@@ -33,6 +33,15 @@ import webserver
 sys.path.remove(_TEST_DIR)
 # pylint: enable=g-import-not-at-top, g-bad-import-order
 
+_VERSION_SPECIFIC_NEGATIVE_FILTER = {}
+
+_VERSION_SPECIFIC_NEGATIVE_FILTER["67"] = [
+    # The wording of an error changed between Chrome 67 and 68 that makes
+    # testCloseWindow fail because we compare the newer error in the log
+    # file to the old error given by Chrome 67 and find a difference.
+    "*testCloseWindow"
+]
+
 
 def SubstituteVariableEntries(s):
   """Identifies and removes items that can legitimately vary between runs."""
@@ -197,6 +206,14 @@ class ChromeDriverClientReplayTest(unittest.TestCase):
   def testChromeBinary(self):
     self.runTest("test_data/testChromeBinary.log")
 
+
+def GetNegativeFilter(chrome_version):
+  if chrome_version in _VERSION_SPECIFIC_FILTER:
+    negative_filter = _VERSION_SPECIFIC_FILTER[chrome_version]
+    return "*-" + ":__main__.".join([""] + negative_filter)
+  return "*"
+
+
 def main():
   usage = "usage: %prog <chromedriver binary> [options]"
   parser = optparse.OptionParser(usage=usage)
@@ -206,11 +223,17 @@ def main():
   parser.add_option(
       "", "--chrome", help="Path to the desired Chrome binary.")
   parser.add_option(
-      "", "--filter", type="string", default="*",
-      help="Filter for specifying what tests to run, \"*\" will run all. E.g., "
-      "*testRunMethod")
+      "", "--chrome-version", default="HEAD",
+      help="Version of Chrome. Default is 'HEAD'.")
+  parser.add_option(
+      "", "--filter", type="string", default="",
+      help="Filter for specifying what tests to run, \"*\" will run all,"
+      "including tests excluded by default. E.g., *testRunMethod")
 
+  # Need global to access these from the test runner.
+  # pylint: disable=global-variable-undefined
   global _OPTIONS, _CHROMEDRIVER
+  # pylint: enable=global-variable-undefined
   _OPTIONS, args = parser.parse_args()
   _CHROMEDRIVER = util.GetAbsolutePathOfUserPath(args[0])
   if not os.path.exists(_CHROMEDRIVER):
@@ -222,9 +245,13 @@ def main():
 
   all_tests_suite = unittest.defaultTestLoader.loadTestsFromModule(
       sys.modules[__name__])
-  tests = unittest_util.FilterTestSuite(all_tests_suite, _OPTIONS.filter)
+
+  test_filter = (GetNegativeFilter(_OPTIONS.chrome_version)
+                 if _OPTIONS.filter == "" else _OPTIONS.filter)
+
+  tests = unittest_util.FilterTestSuite(all_tests_suite, test_filter)
   result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(tests)
   sys.exit(len(result.failures) + len(result.errors))
 
 if __name__ == "__main__":
-  main()
\ No newline at end of file
+  main()

chrome/test/chromedriver/test/run_all_tests.py

@@ -14,6 +14,7 @@ import tempfile
 import traceback
 
 _THIS_DIR = os.path.abspath(os.path.dirname(__file__))
+_PARENT_DIR = os.path.join(_THIS_DIR, os.pardir)
 sys.path.insert(0, os.path.join(_THIS_DIR, os.pardir))
 
 import archive
@@ -57,6 +58,32 @@ def _GenerateTestCommand(script,
   return cmd
 
 
+def RunReplayTests(chromedriver, chrome=None,
+                   chrome_version=None, chrome_version_name=None):
+  version_info = ''
+  if chrome_version_name:
+    version_info = '(%s)' % chrome_version_name
+  util.MarkBuildStepStart('replay_tests%s' % version_info)
+
+  _, log_path = tempfile.mkstemp(prefix='chromedriver_log_')
+  print 'chromedriver server log: %s' % log_path
+  cmd = [
+    sys.executable,
+    os.path.join(_PARENT_DIR, 'log_replay', 'client_replay_test.py'),
+    chromedriver,
+    '--output-log-path=%s' % log_path
+  ]
+  if chrome:
+    cmd.append('--chrome=%s' % chrome)
+  if chrome_version:
+    cmd.append('--chrome-version=%s' % chrome_version)
+  code = util.RunCommand(cmd)
+
+  if code:
+    util.MarkBuildStepError()
+  return code
+
+
 def RunPythonTests(chromedriver, ref_chromedriver,
                    chrome=None, chrome_version=None,
                    chrome_version_name=None, android_package=None):
@@ -231,7 +258,11 @@ def main():
                            chrome_version=version,
                            chrome_version_name='v%s' % version_name,
                            verbose=True)
-      code = code or code1 or code2
+      code3 = RunReplayTests(chromedriver,
+                             chrome=chrome_path,
+                             chrome_version=version,
+                             chrome_version_name='v%s' % version_name)
+      code = code or code1 or code2 or code3
       _KillChromes()
       shutil.rmtree(temp_dir)
     cpp_tests = os.path.join(build_dir, cpp_tests_name)

chrome/test/chromedriver/test/run_py_tests.py

@@ -105,11 +105,6 @@ _VERSION_SPECIFIC_FILTER['69'] = [
     'HeadlessInvalidCertificateTest.*',
 ]
 
-_VERSION_SPECIFIC_FILTER['68'] = []
-
-_VERSION_SPECIFIC_FILTER['67'] = []
-
-
 _OS_SPECIFIC_FILTER = {}
 _OS_SPECIFIC_FILTER['win'] = [
     # https://bugs.chromium.org/p/chromedriver/issues/detail?id=299
@@ -2898,9 +2893,9 @@ if __name__ == '__main__':
       '', '--chrome-version', default='HEAD',
       help='Version of chrome. Default is \'HEAD\'.')
   parser.add_option(
-      '', '--filter', type='string', default='*',
-      help=('Filter for specifying what tests to run, "*" will run all. E.g., '
-            '*testStartStop'))
+      "", "--filter", type="string", default="",
+      help="Filter for specifying what tests to run, \"*\" will run all,"
+      "including tests excluded by default. E.g., *testRunMethod")
   parser.add_option(
       '', '--android-package',
       help=('Android package key. Possible values: ' +
@@ -2958,7 +2953,7 @@ if __name__ == '__main__':
   if _ANDROID_PACKAGE_KEY:
     devil_chromium.Initialize()
 
-  if options.filter == '*':
+  if options.filter == '':
     if _ANDROID_PACKAGE_KEY:
       negative_filter = _ANDROID_NEGATIVE_FILTER[_ANDROID_PACKAGE_KEY]
     else: