Webkit layout test runner: Add option for zero tests executed

Adds --zero-tests-executed-ok, which makes the layout test runner exit
with an ok status if zero tests are executed. This is important for
re-running layout tests without patch, which we do on tryservers to
check if the test was being flaky.

Bug: 533481
Change-Id: I7d6bcd805b99818fdf82ecb88e71863ff72b72e7
Reviewed-on: https://chromium-review.googlesource.com/1018590Reviewed-by: Quinten Yearsley <qyearsley@chromium.org>
Reviewed-by: Jeff Carpenter <jeffcarp@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Stephen Martinis <martiniss@chromium.org>
Cr-Commit-Position: refs/heads/master@{#552426}

testing/buildbot/chromium.fyi.json

@@ -2293,6 +2293,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -2807,6 +2810,9 @@
   "Dummy WebKit Mac10.13": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -4805,6 +4811,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -5324,7 +5333,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--additional-driver-flag=--enable-features=NetworkService"
+          "--additional-driver-flag=--enable-features=NetworkService",
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -7652,7 +7662,8 @@
       {
         "args": [
           "--additional-driver-flag=--enable-blink-features=LayoutNG",
-          "--additional-driver-flag=--enable-slimming-paint-v175"
+          "--additional-driver-flag=--enable-slimming-paint-v175",
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -7680,7 +7691,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--additional-driver-flag=--root-layer-scrolls"
+          "--additional-driver-flag=--root-layer-scrolls",
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -7708,7 +7720,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--additional-driver-flag=--enable-slimming-paint-v2"
+          "--additional-driver-flag=--enable-slimming-paint-v2",
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -8237,8 +8250,9 @@
     "isolated_scripts": [
       {
         "args": [
+          "--additional-driver-flag=--enable-blink-features=HeapIncrementalMarking",
           "--debug",
-          "--additional-driver-flag=--enable-blink-features=HeapIncrementalMarking"
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -8411,6 +8425,9 @@
     ],
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.linux.json

@@ -1386,6 +1386,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.mac.json

@@ -459,6 +459,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -929,6 +932,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -1399,6 +1405,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -1886,6 +1895,9 @@
         }
       },
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.webkit.json

@@ -4,6 +4,9 @@
   "WebKit Linux Trusty": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -29,7 +32,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--debug"
+          "--debug",
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -57,10 +61,11 @@
       {
         "args": [
           "--additional-expectations",
-          "../../third_party/WebKit/LayoutTests/ASANExpectations",
+          "--enable-sanitizer",
           "--time-out-ms",
-          "48000",
-          "--enable-sanitizer"
+          "--zero-tests-executed-ok",
+          "../../third_party/WebKit/LayoutTests/ASANExpectations",
+          "48000"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -88,10 +93,11 @@
       {
         "args": [
           "--additional-expectations",
-          "../../third_party/WebKit/LayoutTests/LeakExpectations",
+          "--enable-leak-detection",
           "--time-out-ms",
-          "48000",
-          "--enable-leak-detection"
+          "--zero-tests-executed-ok",
+          "../../third_party/WebKit/LayoutTests/LeakExpectations",
+          "48000"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -119,10 +125,11 @@
       {
         "args": [
           "--additional-expectations",
-          "../../third_party/WebKit/LayoutTests/MSANExpectations",
+          "--enable-sanitizer",
           "--time-out-ms",
-          "66000",
-          "--enable-sanitizer"
+          "--zero-tests-executed-ok",
+          "../../third_party/WebKit/LayoutTests/MSANExpectations",
+          "66000"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -151,6 +158,9 @@
   "WebKit Mac10.10": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -176,6 +186,9 @@
   "WebKit Mac10.11": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -202,7 +215,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--debug"
+          "--debug",
+          "--zero-tests-executed-ok"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -230,6 +244,9 @@
   "WebKit Mac10.12 (retina)": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -267,6 +284,9 @@
   "WebKit Win10": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -291,6 +311,9 @@
   "WebKit Win7": {
     "isolated_scripts": [
       {
+        "args": [
+          "--zero-tests-executed-ok"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.win.json

@@ -1778,6 +1778,7 @@
         "args": [
           "--debug",
           "--target",
+          "--zero-tests-executed-ok",
           "Debug_x64"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
@@ -3015,6 +3016,7 @@
         "args": [
           "--debug",
           "--time-out-ms",
+          "--zero-tests-executed-ok",
           "60000"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",

testing/buildbot/test_suites.pyl

@@ -543,6 +543,9 @@
 
   'chromium_webkit_isolated_scripts': {
     'webkit_layout_tests': {
+      'args': [
+        '--zero-tests-executed-ok',
+      ],
       'isolate_name': 'webkit_layout_tests_exparchive',
       'merge': {
         'args': [
@@ -818,6 +821,9 @@
     },
     'views_perftests': {},
     'webkit_layout_tests': {
+      'args': [
+        '--zero-tests-executed-ok',
+      ],
       'isolate_name': 'webkit_layout_tests_exparchive',
       'merge': {
         'args': [
@@ -1089,6 +1095,7 @@
     'webkit_layout_tests': {
       'args': [
         '--additional-driver-flag=--enable-features=NetworkService',
+        '--zero-tests-executed-ok',
       ],
       'isolate_name': 'webkit_layout_tests_exparchive',
       'merge': {
@@ -1766,6 +1773,9 @@
   # adding them to test_suite_exceptions.pyl.
   'webkit_layout_tests_isolated_scripts': {
     'webkit_layout_tests': {
+      'args': [
+        '--zero-tests-executed-ok',
+      ],
       'isolate_name': 'webkit_layout_tests_exparchive',
       'merge': {
         'args': [

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/manager.py

@@ -135,8 +135,14 @@ class Manager(object):
 
         # Check to make sure we're not skipping every test.
         if not tests_to_run:
-            _log.critical('No tests to run.')
-            return test_run_results.RunDetails(exit_code=exit_codes.NO_TESTS_EXIT_STATUS)
+            msg = 'No tests to run.'
+            if self._options.zero_tests_executed_ok:
+                _log.info(msg)
+                code = exit_codes.OK_EXIT_STATUS
+            else:
+                _log.critical(msg)
+                code = exit_codes.NO_TESTS_EXIT_STATUS
+            return test_run_results.RunDetails(exit_code=code)
 
         exit_code = self._set_up_run(tests_to_run)
         if exit_code:

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -490,6 +490,11 @@ def parse_args(args):
                 '-w', '--watch',
                 action='store_true',
                 help='Re-run tests quickly (e.g. avoid restarting the server)'),
+            optparse.make_option(
+                '--zero-tests-executed-ok',
+                action='store_true',
+                help='If set, exit with a success code when no tests are run.'
+                ' Used on trybots when layout tests are retried without patch.')
         ]))
 
     # FIXME: Move these into json_results_generator.py.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

@@ -292,6 +292,18 @@ class RunTest(unittest.TestCase, StreamTestingMixin):
         self.assertEqual(details.exit_code, exit_codes.NO_TESTS_EXIT_STATUS)
         self.assert_contains(err, 'No tests to run.\n')
 
+    def test_no_tests_found_with_ok_flag(self):
+        details, err, _ = logging_run(
+            ['resources', '--zero-tests-executed-ok'], tests_included=True)
+        self.assertEqual(details.exit_code, exit_codes.OK_EXIT_STATUS)
+        self.assert_contains(err, 'No tests to run.\n')
+
+    def test_no_tests_found_with_ok_flag_shards(self):
+        details, err, _ = logging_run(
+            ['--shard-index', '4', '--total-shards', '40', 'foo/bar.html', '--zero-tests-executed-ok'], tests_included=True)
+        self.assertEqual(details.exit_code, exit_codes.OK_EXIT_STATUS)
+        self.assert_contains(err, 'No tests to run.\n')
+
     def test_natural_order(self):
         tests_to_run = [
             'passes/audio.html',