Reland "[ChromeDriver] Add ChromeDriver tests to Commit Queue"

This is a reland of aa7860c7,
with the change to enable the ChromeDriver tests on release bot only.
The original commit was reverted due to flakiness of the tests on debug
build. The reason for the flakiness is debug build of Chrome takes a
long time to start, usually between 20 to 40 seconds. The timeout for
the test is 30 seconds, thus it can either succeed or fail depending
on whether Chrome started quick enough or not.

While it is possible to increase the timeout, spending over 30 seconds
per test appears to be too much overhead in the long run. Also, the
purpose of running these tests is to catch incompatibilities that can
develop between Chrome and ChromeDriver, and running these tests on
release builds should suffice for that purpose.

Original change's description:
> [ChromeDriver] Add ChromeDriver tests to Commit Queue
>
> This is the initial check-in of the feature to add ChromeDriver tests
> to commit queue. It includes the following:
>
> * A new test target chromedriver_py_tests, to encapsulate dependencies
>   needed to run the Python-based ChromeDriver tests. This target is
>   currently enabled on Linux only. Other platforms will be added later.
>
> * Only a single test case is enabled. Enabling additional test cases
>   will be done later.
>
> Bug: 793370
> Change-Id: I5a6f0568d2e78724a4fe53e84c3a480d770c3e47
> Reviewed-on: https://chromium-review.googlesource.com/874816
> Commit-Queue: John Chen <johnchen@chromium.org>
> Reviewed-by: John Budorick <jbudorick@chromium.org>
> Reviewed-by: Daniel Cheng <dcheng@chromium.org>
> Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#531234}

Bug: 793370
Change-Id: Ic8873c326dfcd676552083d6f90aba29cc046e67
Reviewed-on: https://chromium-review.googlesource.com/885082Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Commit-Queue: John Chen <johnchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#532252}

PRESUBMIT.py

@@ -522,6 +522,7 @@ _ANDROID_SPECIFIC_PYDEPS_FILES = [
 
 
 _GENERIC_PYDEPS_FILES = [
+    'chrome/test/chromedriver/test/run_py_tests.pydeps',
 ]
 
 
@@ -2089,9 +2090,9 @@ class PydepsChecker(object):
 
 def _CheckPydepsNeedsUpdating(input_api, output_api, checker_for_tests=None):
   """Checks if a .pydeps file needs to be regenerated."""
-  # This check is mainly for Android, and involves paths not only in the
-  # PRESUBMIT.py, but also in the .pydeps files. It doesn't work on Windows and
-  # Mac, so skip it on other platforms.
+  # This check is for Python dependency lists (.pydeps files), and involves
+  # paths not only in the PRESUBMIT.py, but also in the .pydeps files. It
+  # doesn't work on Windows and Mac, so skip it on other platforms.
   if input_api.platform != 'linux2':
     return []
   # TODO(agrieve): Update when there's a better way to detect

chrome/test/chromedriver/BUILD.gn

@@ -326,6 +326,29 @@ executable("chromedriver") {
   ]
 }
 
+group("chromedriver_py_tests") {
+  testonly = true
+  deps = [
+    ":chromedriver",
+    "//chrome:chrome",
+  ]
+
+  _py_files = read_file("test/run_py_tests.pydeps", "list lines")
+
+  # Filter out comments.
+  set_sources_assignment_filter([ "#*" ])
+  sources = _py_files
+
+  data = sources + [
+           "//chrome/test/data/chromedriver/",
+           "//testing/xvfb.py",
+         ]
+
+  data_deps = [
+    "//chrome/test/chromedriver",
+  ]
+}
+
 test("chromedriver_unittests") {
   sources = [
     "capabilities_unittest.cc",

chrome/test/chromedriver/test/run_py_tests.py

@@ -2706,6 +2706,14 @@ if __name__ == '__main__':
       '', '--android-package',
       help=('Android package key. Possible values: ' +
             str(_ANDROID_NEGATIVE_FILTER.keys())))
+
+  parser.add_option(
+      '', '--isolated-script-test-output',
+      help='JSON output file used by swarming')
+  parser.add_option(
+      '', '--isolated-script-test-perf-output',
+      help='JSON perf output file used by swarming, ignored')
+
   options, args = parser.parse_args()
 
   options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver)
@@ -2757,4 +2765,32 @@ if __name__ == '__main__':
   ChromeDriverTest.GlobalTearDown()
   HeadlessInvalidCertificateTest.GlobalTearDown()
   MobileEmulationCapabilityTest.GlobalTearDown()
+
+  if options.isolated_script_test_output:
+    output = {
+        'interrupted': False,
+        'num_failures_by_type': { },
+        'path_delimiter': '.',
+        'seconds_since_epoch': time.time(),
+        'tests': { },
+        'version': 3,
+    }
+
+    for test in tests:
+      output['tests'][test.id()] = {
+          'expected': 'PASS',
+          'actual': 'PASS'
+      }
+
+    for failure in result.failures + result.errors:
+      output['tests'][failure[0].id()]['actual'] = 'FAIL'
+
+    num_fails = len(result.failures) + len(result.errors)
+    output['num_failures_by_type']['FAIL'] = num_fails
+    output['num_failures_by_type']['PASS'] = len(output['tests']) - num_fails
+
+    with open(options.isolated_script_test_output, 'w') as fp:
+      json.dump(output, fp)
+      fp.write('\n')
+
   sys.exit(len(result.failures) + len(result.errors))

chrome/test/chromedriver/test/run_py_tests.pydeps

+# Generated by running:
+#   build/print_python_deps.py --root chrome/test/chromedriver --output chrome/test/chromedriver/test/run_py_tests.pydeps chrome/test/chromedriver/test/run_py_tests.py
+../../../build/android/devil_chromium.py
+../../../build/android/pylib/__init__.py
+../../../build/android/pylib/constants/__init__.py
+../../../build/android/pylib/constants/host_paths.py
+../../../third_party/catapult/common/py_utils/py_utils/__init__.py
+../../../third_party/catapult/common/py_utils/py_utils/cloud_storage.py
+../../../third_party/catapult/common/py_utils/py_utils/cloud_storage_global_lock.py
+../../../third_party/catapult/common/py_utils/py_utils/lock.py
+../../../third_party/catapult/common/py_utils/py_utils/tempfile_ext.py
+../../../third_party/catapult/dependency_manager/dependency_manager/__init__.py
+../../../third_party/catapult/dependency_manager/dependency_manager/archive_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/base_config.py
+../../../third_party/catapult/dependency_manager/dependency_manager/cloud_storage_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/dependency_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/dependency_manager_util.py
+../../../third_party/catapult/dependency_manager/dependency_manager/exceptions.py
+../../../third_party/catapult/dependency_manager/dependency_manager/local_path_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/manager.py
+../../../third_party/catapult/dependency_manager/dependency_manager/uploader.py
+../../../third_party/catapult/devil/devil/__init__.py
+../../../third_party/catapult/devil/devil/android/__init__.py
+../../../third_party/catapult/devil/devil/android/apk_helper.py
+../../../third_party/catapult/devil/devil/android/constants/__init__.py
+../../../third_party/catapult/devil/devil/android/constants/chrome.py
+../../../third_party/catapult/devil/devil/android/constants/file_system.py
+../../../third_party/catapult/devil/devil/android/decorators.py
+../../../third_party/catapult/devil/devil/android/device_errors.py
+../../../third_party/catapult/devil/devil/android/device_signal.py
+../../../third_party/catapult/devil/devil/android/device_temp_file.py
+../../../third_party/catapult/devil/devil/android/device_utils.py
+../../../third_party/catapult/devil/devil/android/forwarder.py
+../../../third_party/catapult/devil/devil/android/install_commands.py
+../../../third_party/catapult/devil/devil/android/logcat_monitor.py
+../../../third_party/catapult/devil/devil/android/md5sum.py
+../../../third_party/catapult/devil/devil/android/sdk/__init__.py
+../../../third_party/catapult/devil/devil/android/sdk/aapt.py
+../../../third_party/catapult/devil/devil/android/sdk/adb_wrapper.py
+../../../third_party/catapult/devil/devil/android/sdk/build_tools.py
+../../../third_party/catapult/devil/devil/android/sdk/intent.py
+../../../third_party/catapult/devil/devil/android/sdk/keyevent.py
+../../../third_party/catapult/devil/devil/android/sdk/split_select.py
+../../../third_party/catapult/devil/devil/android/sdk/version_codes.py
+../../../third_party/catapult/devil/devil/android/valgrind_tools/__init__.py
+../../../third_party/catapult/devil/devil/android/valgrind_tools/base_tool.py
+../../../third_party/catapult/devil/devil/base_error.py
+../../../third_party/catapult/devil/devil/constants/__init__.py
+../../../third_party/catapult/devil/devil/constants/exit_codes.py
+../../../third_party/catapult/devil/devil/devil_env.py
+../../../third_party/catapult/devil/devil/utils/__init__.py
+../../../third_party/catapult/devil/devil/utils/cmd_helper.py
+../../../third_party/catapult/devil/devil/utils/host_utils.py
+../../../third_party/catapult/devil/devil/utils/lazy/__init__.py
+../../../third_party/catapult/devil/devil/utils/lazy/weak_constant.py
+../../../third_party/catapult/devil/devil/utils/parallelizer.py
+../../../third_party/catapult/devil/devil/utils/reraiser_thread.py
+../../../third_party/catapult/devil/devil/utils/timeout_retry.py
+../../../third_party/catapult/devil/devil/utils/watchdog_timer.py
+../../../third_party/catapult/devil/devil/utils/zip_utils.py
+../../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py
+chrome_paths.py
+client/chromedriver.py
+client/command_executor.py
+client/webelement.py
+server/server.py
+test/run_py_tests.py
+test/unittest_util.py
+test/webserver.py
+util.py

testing/buildbot/chromium.linux.json

@@ -1189,6 +1189,16 @@
       }
     ],
     "isolated_scripts": [
+      {
+        "args": [
+          "--filter=__main__.ChromeDriverTest.testLoadUrl"
+        ],
+        "isolate_name": "chromedriver_py_tests",
+        "name": "chromedriver_py_tests",
+        "swarming": {
+          "can_use_on_swarming_builders": true
+        }
+      },
       {
         "isolate_name": "content_shell_crash_test",
         "name": "content_shell_crash_test",

testing/buildbot/gn_isolate_map.pyl

@@ -311,6 +311,16 @@
     "label": "//chrome/test/chromedriver:chromedriver",
     "type": "additional_compile_target",
   },
+  "chromedriver_py_tests": {
+    "label": "//chrome/test/chromedriver:chromedriver_py_tests",
+    "type": "script",
+    "script": "//testing/xvfb.py",
+    "args": [
+      "../../chrome/test/chromedriver/test/run_py_tests.py",
+      "--chromedriver=chromedriver",
+      "--log-path=${ISOLATED_OUTDIR}/chromedriver.log",
+    ],
+  },
   "chromedriver_unittests": {
     "label": "//chrome/test/chromedriver:chromedriver_unittests",
     "type": "windowed_test_launcher",

testing/buildbot/test_suites.pyl

@@ -327,6 +327,18 @@
     },
   },
 
+  'chromedriver_py_tests_isolated_scripts': {
+    'chromedriver_py_tests': {
+      "args": [
+        # TODO(johnchen@chromium.org): As the initial step of adding
+        # ChromeDriver tests to commit queue, the following filter is used to
+        # enable a single test case as a smoke test. Many more test cases will
+        # be enabled in the future.
+        "--filter=__main__.ChromeDriverTest.testLoadUrl",
+      ],
+    },
+  },
+
   'chromium_android_asan_gtests': {
     # TODO(kbr): reduce duplication among these tests, and with other
     # test suites.
@@ -1947,6 +1959,13 @@
     'telemetry_perf_unittests_isolated_scripts',
   ],
 
+  'chromium_linux_rel_isolated_scripts': [
+    'chromedriver_py_tests_isolated_scripts',
+    'desktop_chromium_isolated_scripts',
+    'linux_specific_chromium_isolated_scripts',
+    'telemetry_perf_unittests_isolated_scripts',
+  ],
+
   'chromium_mac_clang_gtests': [
     'angle_gtests',
     'chromium_gtests',

testing/buildbot/waterfalls.pyl

@@ -1394,7 +1394,7 @@
       'Linux Tests': {
         'test_suites': {
           'gtest_tests': 'chromium_linux_gtests',
-          'isolated_scripts': 'chromium_linux_isolated_scripts',
+          'isolated_scripts': 'chromium_linux_rel_isolated_scripts',
           'scripts': 'chromium_linux_scripts',
         },
       },