[ChromeDriver] Add ability to run tests with @NotYetImplemented annotation

related CL: https://chromium-review.googlesource.com/c/chromium/deps/webdriver/+/1287025

Change-Id: I7e18664b01de7ed627eecec4f7cedac0eeb39079
Reviewed-on: https://chromium-review.googlesource.com/c/1287186Reviewed-by: John Chen <johnchen@chromium.org>
Commit-Queue: Tatiana Buldina <buldina@chromium.org>
Cr-Commit-Position: refs/heads/master@{#606598}

chrome/test/chromedriver/test/run_java_tests.py

@@ -64,7 +64,7 @@ class TestResult(object):
     return self._failure
 
 
-def _Run(java_tests_src_dir, test_filter,
+def _Run(java_tests_src_dir, test_filter, ready_to_run_tests,
          chromedriver_path, chrome_path, log_path, android_package_key,
          verbose, debug):
   """Run the WebDriver Java tests and return the test results.
@@ -73,6 +73,8 @@ def _Run(java_tests_src_dir, test_filter,
     java_tests_src_dir: the java test source code directory.
     test_filter: the filter to use when choosing tests to run. Format is same
         as Google C++ Test format.
+    readyToRunTests: tests that need to run regardless of
+        @NotYetImplemented annotation
     chromedriver_path: path to ChromeDriver exe.
     chrome_path: path to Chrome exe.
     log_path: path to server log.
@@ -124,6 +126,8 @@ def _Run(java_tests_src_dir, test_filter,
     # Test jar actually takes a regex. Convert from glob.
     test_filter = test_filter.replace('*', '.*')
     sys_props += ['filter=' + test_filter]
+  if ready_to_run_tests:
+    sys_props += ['readyToRun=' + ready_to_run_tests]
 
   jvm_args = []
   if debug:
@@ -313,6 +317,7 @@ def main():
           test_filter += '-'
         test_filter += ':'.join(environment.GetDisabledJavaTestMatchers())
       test_filters = [test_filter]
+    ready_to_run_tests = ':'.join(environment.GetReadyToRunJavaTestMatchers())
 
     java_tests_src_dir = os.path.join(chrome_paths.GetSrc(), 'chrome', 'test',
                                       'chromedriver', 'third_party',
@@ -334,6 +339,7 @@ def main():
       results += _Run(
           java_tests_src_dir=java_tests_src_dir,
           test_filter=filter,
+          ready_to_run_tests=ready_to_run_tests,
           chromedriver_path=options.chromedriver,
           chrome_path=util.GetAbsolutePathOfUserPath(options.chrome),
           log_path=options.log_path,

chrome/test/chromedriver/test/test_environment.py

@@ -66,6 +66,17 @@ class BaseTestEnvironment(object):
     return _EXPECTATIONS['GetDisabledTestMatchers'](
         self.GetOS(), self._chrome_version)
 
+  def GetReadyToRunJavaTestMatchers(self):
+    """Get the list of disabled for Chrome java test matchers
+    but which already works.
+
+    Returns:
+      List of disabled for Chrome java test matchers
+      but which already works.
+    """
+    return _EXPECTATIONS['GetReadyToRunTestMatchers'](
+        self._chrome_version)
+
   def GetPassedJavaTests(self):
     """Get the list of passed java tests.
 

chrome/test/chromedriver/test/test_expectations

@@ -12,14 +12,6 @@ import fnmatch
 
 _REVISION_NEGATIVE_FILTER = {}
 _REVISION_NEGATIVE_FILTER['HEAD'] = [
-    # marked as not yet implemented with chrome but already works
-    'ChildrenFindingTest.testShouldNotReturnRootElementWhenFindingChildrenById',
-    'ClearTest.shouldBeAbleToClearDateInput',
-    'ClearTest.shouldBeAbleToClearDatetimeLocalInput',
-    'ClearTest.shouldBeAbleToClearMonthInput',
-    'ClearTest.shouldBeAbleToClearTimeInput',
-    'ClearTest.shouldBeAbleToClearWeekInput',
-
     # Flaky: https://bugs.chromium.org/p/chromedriver/issues/detail?id=528
     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
 
@@ -31,6 +23,17 @@ _REVISION_NEGATIVE_FILTER['HEAD'] = [
     'ReferrerTest.navigationWhenProxyInterceptsASpecificUrl',
 ]
 
+_REVISION_READY_TO_RUN_FILTER = {}
+_REVISION_READY_TO_RUN_FILTER['HEAD'] = [
+    # marked as not yet implemented with chrome but already works
+    'ChildrenFindingTest.testShouldNotReturnRootElementWhenFindingChildrenById',
+    'ClearTest.shouldBeAbleToClearDateInput',
+    'ClearTest.shouldBeAbleToClearDatetimeLocalInput',
+    'ClearTest.shouldBeAbleToClearMonthInput',
+    'ClearTest.shouldBeAbleToClearTimeInput',
+    'ClearTest.shouldBeAbleToClearWeekInput',
+]
+
 _OS_NEGATIVE_FILTER = {}
 _OS_NEGATIVE_FILTER['win'] = [
 ]
@@ -221,6 +224,11 @@ def _GetRevisionNegativeFilter(chrome_version):
     return _REVISION_NEGATIVE_FILTER[chrome_version]
   return _REVISION_NEGATIVE_FILTER['HEAD']
 
+def _GetRevisionReadyToRunFilter(chrome_version):
+  if chrome_version in _REVISION_READY_TO_RUN_FILTER:
+    return _REVISION_READY_TO_RUN_FILTER[chrome_version]
+  return _REVISION_READY_TO_RUN_FILTER['HEAD']
+
 def _GetSpecificOsRevisionNegativeFilter(operating_system, chrome_version):
   os_version = operating_system + "_" + chrome_version
   if os_version in _SPECIFIC_OS_REVISION_NEGATIVE_FILTER:
@@ -244,6 +252,8 @@ def GetDisabledTestMatchers(operating_system, chrome_version):
           _GetSpecificOsRevisionNegativeFilter(operating_system,
                                                chrome_version))[:]
 
+def GetReadyToRunTestMatchers(chrome_version):
+  return (_GetRevisionReadyToRunFilter(chrome_version))[:]
 
 def ApplyJavaTestFilter(operating_system, chrome_version, tests):
   """Applies the test filter to the given list of tests.