[ChromeDriver] Add ChromeDriver tests to Commit Queue

This is the initial check-in of the feature to add ChromeDriver tests
to commit queue. It includes the following:

* A new test target chromedriver_py_tests, to encapsulate dependencies
  needed to run the Python-based ChromeDriver tests. This target is
  currently enabled on Linux only. Other platforms will be added later.

* Only a single test case is enabled. Enabling additional test cases
  will be done later.

Bug: 793370
Change-Id: I5a6f0568d2e78724a4fe53e84c3a480d770c3e47
Reviewed-on: https://chromium-review.googlesource.com/874816
Commit-Queue: John Chen <johnchen@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#531234}

PRESUBMIT.py

@@ -521,6 +521,7 @@ _ANDROID_SPECIFIC_PYDEPS_FILES = [
 
 
 _GENERIC_PYDEPS_FILES = [
+    'chrome/test/chromedriver/test/run_py_tests.pydeps',
 ]
 
 
@@ -2081,9 +2082,9 @@ class PydepsChecker(object):
 
 def _CheckPydepsNeedsUpdating(input_api, output_api, checker_for_tests=None):
   """Checks if a .pydeps file needs to be regenerated."""
-  # This check is mainly for Android, and involves paths not only in the
-  # PRESUBMIT.py, but also in the .pydeps files. It doesn't work on Windows and
-  # Mac, so skip it on other platforms.
+  # This check is for Python dependency lists (.pydeps files), and involves
+  # paths not only in the PRESUBMIT.py, but also in the .pydeps files. It
+  # doesn't work on Windows and Mac, so skip it on other platforms.
   if input_api.platform != 'linux2':
     return []
   # TODO(agrieve): Update when there's a better way to detect

chrome/test/chromedriver/BUILD.gn

@@ -326,6 +326,29 @@ executable("chromedriver") {
   ]
 }
 
+group("chromedriver_py_tests") {
+  testonly = true
+  deps = [
+    ":chromedriver",
+    "//chrome:chrome",
+  ]
+
+  _py_files = read_file("test/run_py_tests.pydeps", "list lines")
+
+  # Filter out comments.
+  set_sources_assignment_filter([ "#*" ])
+  sources = _py_files
+
+  data = sources + [
+           "//chrome/test/data/chromedriver/",
+           "//testing/xvfb.py",
+         ]
+
+  data_deps = [
+    "//chrome/test/chromedriver",
+  ]
+}
+
 test("chromedriver_unittests") {
   sources = [
     "capabilities_unittest.cc",

chrome/test/chromedriver/test/run_py_tests.py

@@ -2701,6 +2701,14 @@ if __name__ == '__main__':
       '', '--android-package',
       help=('Android package key. Possible values: ' +
             str(_ANDROID_NEGATIVE_FILTER.keys())))
+
+  parser.add_option(
+      '', '--isolated-script-test-output',
+      help='JSON output file used by swarming')
+  parser.add_option(
+      '', '--isolated-script-test-perf-output',
+      help='JSON perf output file used by swarming, ignored')
+
   options, args = parser.parse_args()
 
   options.chromedriver = util.GetAbsolutePathOfUserPath(options.chromedriver)
@@ -2752,4 +2760,32 @@ if __name__ == '__main__':
   ChromeDriverTest.GlobalTearDown()
   HeadlessInvalidCertificateTest.GlobalTearDown()
   MobileEmulationCapabilityTest.GlobalTearDown()
+
+  if options.isolated_script_test_output:
+    output = {
+        'interrupted': False,
+        'num_failures_by_type': { },
+        'path_delimiter': '.',
+        'seconds_since_epoch': time.time(),
+        'tests': { },
+        'version': 3,
+    }
+
+    for test in tests:
+      output['tests'][test.id()] = {
+          'expected': 'PASS',
+          'actual': 'PASS'
+      }
+
+    for failure in result.failures + result.errors:
+      output['tests'][failure[0].id()]['actual'] = 'FAIL'
+
+    num_fails = len(result.failures) + len(result.errors)
+    output['num_failures_by_type']['FAIL'] = num_fails
+    output['num_failures_by_type']['PASS'] = len(output['tests']) - num_fails
+
+    with open(options.isolated_script_test_output, 'w') as fp:
+      json.dump(output, fp)
+      fp.write('\n')
+
   sys.exit(len(result.failures) + len(result.errors))

chrome/test/chromedriver/test/run_py_tests.pydeps

+# Generated by running:
+#   build/print_python_deps.py --root chrome/test/chromedriver --output chrome/test/chromedriver/test/run_py_tests.pydeps chrome/test/chromedriver/test/run_py_tests.py
+../../../build/android/devil_chromium.py
+../../../build/android/pylib/__init__.py
+../../../build/android/pylib/constants/__init__.py
+../../../build/android/pylib/constants/host_paths.py
+../../../third_party/catapult/common/py_utils/py_utils/__init__.py
+../../../third_party/catapult/common/py_utils/py_utils/cloud_storage.py
+../../../third_party/catapult/common/py_utils/py_utils/cloud_storage_global_lock.py
+../../../third_party/catapult/common/py_utils/py_utils/lock.py
+../../../third_party/catapult/common/py_utils/py_utils/tempfile_ext.py
+../../../third_party/catapult/dependency_manager/dependency_manager/__init__.py
+../../../third_party/catapult/dependency_manager/dependency_manager/archive_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/base_config.py
+../../../third_party/catapult/dependency_manager/dependency_manager/cloud_storage_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/dependency_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/dependency_manager_util.py
+../../../third_party/catapult/dependency_manager/dependency_manager/exceptions.py
+../../../third_party/catapult/dependency_manager/dependency_manager/local_path_info.py
+../../../third_party/catapult/dependency_manager/dependency_manager/manager.py
+../../../third_party/catapult/dependency_manager/dependency_manager/uploader.py
+../../../third_party/catapult/devil/devil/__init__.py
+../../../third_party/catapult/devil/devil/android/__init__.py
+../../../third_party/catapult/devil/devil/android/apk_helper.py
+../../../third_party/catapult/devil/devil/android/constants/__init__.py
+../../../third_party/catapult/devil/devil/android/constants/chrome.py
+../../../third_party/catapult/devil/devil/android/constants/file_system.py
+../../../third_party/catapult/devil/devil/android/decorators.py
+../../../third_party/catapult/devil/devil/android/device_errors.py
+../../../third_party/catapult/devil/devil/android/device_signal.py
+../../../third_party/catapult/devil/devil/android/device_temp_file.py
+../../../third_party/catapult/devil/devil/android/device_utils.py
+../../../third_party/catapult/devil/devil/android/forwarder.py
+../../../third_party/catapult/devil/devil/android/install_commands.py
+../../../third_party/catapult/devil/devil/android/logcat_monitor.py
+../../../third_party/catapult/devil/devil/android/md5sum.py
+../../../third_party/catapult/devil/devil/android/sdk/__init__.py
+../../../third_party/catapult/devil/devil/android/sdk/aapt.py
+../../../third_party/catapult/devil/devil/android/sdk/adb_wrapper.py
+../../../third_party/catapult/devil/devil/android/sdk/build_tools.py
+../../../third_party/catapult/devil/devil/android/sdk/intent.py
+../../../third_party/catapult/devil/devil/android/sdk/keyevent.py
+../../../third_party/catapult/devil/devil/android/sdk/split_select.py
+../../../third_party/catapult/devil/devil/android/sdk/version_codes.py
+../../../third_party/catapult/devil/devil/android/valgrind_tools/__init__.py
+../../../third_party/catapult/devil/devil/android/valgrind_tools/base_tool.py
+../../../third_party/catapult/devil/devil/base_error.py
+../../../third_party/catapult/devil/devil/constants/__init__.py
+../../../third_party/catapult/devil/devil/constants/exit_codes.py
+../../../third_party/catapult/devil/devil/devil_env.py
+../../../third_party/catapult/devil/devil/utils/__init__.py
+../../../third_party/catapult/devil/devil/utils/cmd_helper.py
+../../../third_party/catapult/devil/devil/utils/host_utils.py
+../../../third_party/catapult/devil/devil/utils/lazy/__init__.py
+../../../third_party/catapult/devil/devil/utils/lazy/weak_constant.py
+../../../third_party/catapult/devil/devil/utils/parallelizer.py
+../../../third_party/catapult/devil/devil/utils/reraiser_thread.py
+../../../third_party/catapult/devil/devil/utils/timeout_retry.py
+../../../third_party/catapult/devil/devil/utils/watchdog_timer.py
+../../../third_party/catapult/devil/devil/utils/zip_utils.py
+../../../third_party/catapult/third_party/zipfile/zipfile_2_7_13.py
+chrome_paths.py
+client/chromedriver.py
+client/command_executor.py
+client/webelement.py
+server/server.py
+test/run_py_tests.py
+test/unittest_util.py
+test/webserver.py
+util.py

testing/buildbot/chromium.linux.json

@@ -1189,6 +1189,16 @@
       }
     ],
     "isolated_scripts": [
+      {
+        "args": [
+          "--filter=__main__.ChromeDriverTest.testLoadUrl"
+        ],
+        "isolate_name": "chromedriver_py_tests",
+        "name": "chromedriver_py_tests",
+        "swarming": {
+          "can_use_on_swarming_builders": true
+        }
+      },
       {
         "isolate_name": "content_shell_crash_test",
         "name": "content_shell_crash_test",
@@ -1939,6 +1949,16 @@
       }
     ],
     "isolated_scripts": [
+      {
+        "args": [
+          "--filter=__main__.ChromeDriverTest.testLoadUrl"
+        ],
+        "isolate_name": "chromedriver_py_tests",
+        "name": "chromedriver_py_tests",
+        "swarming": {
+          "can_use_on_swarming_builders": true
+        }
+      },
       {
         "isolate_name": "metrics_python_tests",
         "name": "metrics_python_tests",
@@ -2431,6 +2451,16 @@
       }
     ],
     "isolated_scripts": [
+      {
+        "args": [
+          "--filter=__main__.ChromeDriverTest.testLoadUrl"
+        ],
+        "isolate_name": "chromedriver_py_tests",
+        "name": "chromedriver_py_tests",
+        "swarming": {
+          "can_use_on_swarming_builders": true
+        }
+      },
       {
         "isolate_name": "metrics_python_tests",
         "name": "metrics_python_tests",

testing/buildbot/chromium.sandbox.json

@@ -575,6 +575,16 @@
       }
     ],
     "isolated_scripts": [
+      {
+        "args": [
+          "--filter=__main__.ChromeDriverTest.testLoadUrl"
+        ],
+        "isolate_name": "chromedriver_py_tests",
+        "name": "chromedriver_py_tests",
+        "swarming": {
+          "can_use_on_swarming_builders": true
+        }
+      },
       {
         "isolate_name": "content_shell_crash_test",
         "name": "content_shell_crash_test",

testing/buildbot/gn_isolate_map.pyl

@@ -311,6 +311,16 @@
     "label": "//chrome/test/chromedriver:chromedriver",
     "type": "additional_compile_target",
   },
+  "chromedriver_py_tests": {
+    "label": "//chrome/test/chromedriver:chromedriver_py_tests",
+    "type": "script",
+    "script": "//testing/xvfb.py",
+    "args": [
+      "../../chrome/test/chromedriver/test/run_py_tests.py",
+      "--chromedriver=chromedriver",
+      "--log-path=${ISOLATED_OUTDIR}/chromedriver.log",
+    ],
+  },
   "chromedriver_unittests": {
     "label": "//chrome/test/chromedriver:chromedriver_unittests",
     "type": "windowed_test_launcher",

testing/buildbot/test_suites.pyl

@@ -327,6 +327,18 @@
     },
   },
 
+  'chromedriver_py_tests_isolated_scripts': {
+    'chromedriver_py_tests': {
+      "args": [
+        # TODO(johnchen@chromium.org): As the initial step of adding
+        # ChromeDriver tests to commit queue, the following filter is used to
+        # enable a single test case as a smoke test. Many more test cases will
+        # be enabled in the future.
+        "--filter=__main__.ChromeDriverTest.testLoadUrl",
+      ],
+    },
+  },
+
   'chromium_android_asan_gtests': {
     # TODO(kbr): reduce duplication among these tests, and with other
     # test suites.
@@ -1940,6 +1952,7 @@
   ],
 
   'chromium_linux_isolated_scripts': [
+    'chromedriver_py_tests_isolated_scripts',
     'desktop_chromium_isolated_scripts',
     'linux_specific_chromium_isolated_scripts',
     'telemetry_perf_unittests_isolated_scripts',