Always retry webkit_layout_test failures 3 times.

By default, webkit_layout_test failurse are retried 3 times. But when
'--test-list' is passed, the retry count defaults to 0. To simplify the logic,
this CL makes it so that webkit_layout_test failures are always retried 3 times.

Bug: 888734
Change-Id: I46d15d71c979e5f12f334f5dcff41637f3f3aef4
Reviewed-on: https://chromium-review.googlesource.com/1244018
Commit-Queue: Erik Chen <erikchen@chromium.org>
Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Cr-Commit-Position: refs/heads/master@{#594383}

testing/buildbot/chromium.android.json

@@ -3184,6 +3184,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--debug",
           "--android",
           "--additional-driver-flag=--use-gpu-in-tests"

testing/buildbot/chromium.clang.json

@@ -13808,6 +13808,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.fyi.json

@@ -1692,7 +1692,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--additional-driver-flag=--enable-features=NetworkService"
+          "--additional-driver-flag=--enable-features=NetworkService",
+          "--num-retries=3"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -1823,7 +1824,8 @@
     "isolated_scripts": [
       {
         "args": [
-          "--additional-driver-flag=--enable-features=NetworkService"
+          "--additional-driver-flag=--enable-features=NetworkService",
+          "--num-retries=3"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -1973,6 +1975,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--additional-driver-flag=--enable-blink-features=LayoutNG",
           "--additional-driver-flag=--enable-slimming-paint-v175"
         ],
@@ -2002,6 +2005,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--additional-driver-flag=--root-layer-scrolls"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
@@ -2030,6 +2034,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--additional-driver-flag=--enable-slimming-paint-v2"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
@@ -3919,6 +3924,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--debug"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
@@ -3941,6 +3947,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--debug",
           "--additional-driver-flag=--enable-blink-gen-property-trees"
         ],
@@ -4216,6 +4223,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--debug",
           "--additional-driver-flag=--enable-blink-features=HeapIncrementalMarkingStress"
         ],
@@ -4613,6 +4621,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--debug",
           "--additional-driver-flag=--enable-blink-features=HeapUnifiedGarbageCollection"
         ],
@@ -4888,6 +4897,9 @@
     ],
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -4907,6 +4919,9 @@
   "linux-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -5578,7 +5593,8 @@
           "--additional-driver-flag",
           "--isolate-origins=http://www.web-platform.test:8001/,http://www1.web-platform.test:8001/,http://www2.web-platform.test:8001/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001/,http://xn--lve-6lad.web-platform.test:8001/,http://www.web-platform.test:8081/,http://www1.web-platform.test:8081/,http://www2.web-platform.test:8081/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8081/,http://xn--lve-6lad.web-platform.test:8081/,https://www.web-platform.test:8444/,https://www1.web-platform.test:8444/,https://www2.web-platform.test:8444/,https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8444/,https://xn--lve-6lad.web-platform.test:8444/",
           "--additional-expectations",
-          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process"
+          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process",
+          "--num-retries=3"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -5642,6 +5658,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -6304,7 +6323,8 @@
           "--additional-driver-flag",
           "--isolate-origins=http://www.web-platform.test:8001/,http://www1.web-platform.test:8001/,http://www2.web-platform.test:8001/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001/,http://xn--lve-6lad.web-platform.test:8001/,http://www.web-platform.test:8081/,http://www1.web-platform.test:8081/,http://www2.web-platform.test:8081/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8081/,http://xn--lve-6lad.web-platform.test:8081/,https://www.web-platform.test:8444/,https://www1.web-platform.test:8444/,https://www2.web-platform.test:8444/,https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8444/,https://xn--lve-6lad.web-platform.test:8444/",
           "--additional-expectations",
-          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process"
+          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process",
+          "--num-retries=3"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -6364,6 +6384,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7118,6 +7141,9 @@
   "mac10.10-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7143,6 +7169,9 @@
   "mac10.11-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7168,6 +7197,9 @@
   "mac10.12-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7194,6 +7226,9 @@
   "mac10.13-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7221,6 +7256,9 @@
   "mac10.13_retina-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7256,6 +7294,9 @@
   "win10-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -7281,6 +7322,9 @@
   "win7-blink-rel-dummy": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.linux.json

@@ -1511,7 +1511,8 @@
           "--additional-driver-flag",
           "--isolate-origins=http://www.web-platform.test:8001/,http://www1.web-platform.test:8001/,http://www2.web-platform.test:8001/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001/,http://xn--lve-6lad.web-platform.test:8001/,http://www.web-platform.test:8081/,http://www1.web-platform.test:8081/,http://www2.web-platform.test:8081/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8081/,http://xn--lve-6lad.web-platform.test:8081/,https://www.web-platform.test:8444/,https://www1.web-platform.test:8444/,https://www2.web-platform.test:8444/,https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8444/,https://xn--lve-6lad.web-platform.test:8444/",
           "--additional-expectations",
-          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process"
+          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process",
+          "--num-retries=3"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -1575,6 +1576,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -2277,6 +2281,7 @@
       },
       {
         "args": [
+          "--num-retries=3",
           "--debug"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
@@ -4097,7 +4102,8 @@
           "--additional-driver-flag",
           "--isolate-origins=http://www.web-platform.test:8001/,http://www1.web-platform.test:8001/,http://www2.web-platform.test:8001/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001/,http://xn--lve-6lad.web-platform.test:8001/,http://www.web-platform.test:8081/,http://www1.web-platform.test:8081/,http://www2.web-platform.test:8081/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8081/,http://xn--lve-6lad.web-platform.test:8081/,https://www.web-platform.test:8444/,https://www1.web-platform.test:8444/,https://www2.web-platform.test:8444/,https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8444/,https://xn--lve-6lad.web-platform.test:8444/",
           "--additional-expectations",
-          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process"
+          "src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process",
+          "--num-retries=3"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
@@ -4181,6 +4187,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.mac.json

@@ -1097,6 +1097,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -2241,6 +2244,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -3470,6 +3476,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -4596,6 +4605,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -5713,6 +5725,7 @@
       },
       {
         "args": [
+          "--num-retries=3",
           "--debug"
         ],
         "isolate_name": "webkit_layout_tests_exparchive",

testing/buildbot/chromium.webkit.json

@@ -5,6 +5,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--additional-expectations",
           "../../third_party/WebKit/LayoutTests/ASANExpectations",
           "--time-out-ms",
@@ -36,6 +37,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--additional-expectations",
           "../../third_party/WebKit/LayoutTests/LeakExpectations",
           "--time-out-ms",
@@ -67,6 +69,7 @@
     "isolated_scripts": [
       {
         "args": [
+          "--num-retries=3",
           "--additional-expectations",
           "../../third_party/WebKit/LayoutTests/MSANExpectations",
           "--time-out-ms",
@@ -100,6 +103,9 @@
   "WebKit Mac10.13 (retina)": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -127,6 +133,9 @@
   "WebKit Win10": {
     "isolated_scripts": [
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [

testing/buildbot/chromium.win.json

@@ -4069,6 +4069,9 @@
         }
       },
       {
+        "args": [
+          "--num-retries=3"
+        ],
         "isolate_name": "webkit_layout_tests_exparchive",
         "merge": {
           "args": [
@@ -4840,6 +4843,7 @@
       },
       {
         "args": [
+          "--num-retries=3",
           "--debug",
           "--time-out-ms",
           "60000"

testing/buildbot/test_suites.pyl

@@ -806,6 +806,12 @@
 
     'chromium_webkit_isolated_scripts': {
       'webkit_layout_tests': {
+        # layout test failures are retried 3 times when '--test-list' is not
+        # passed, but 0 times when '--test-list' is passed. We want to always
+        # retry 3 times, so we explicitly specify it.
+        'args': [
+          '--num-retries=3',
+        ],
         'isolate_name': 'webkit_layout_tests_exparchive',
         'merge': {
           'args': [
@@ -935,6 +941,12 @@
       },
       'views_perftests': {},
       'webkit_layout_tests': {
+        # layout test failures are retried 3 times when '--test-list' is not
+        # passed, but 0 times when '--test-list' is passed. We want to always
+        # retry 3 times, so we explicitly specify it.
+        'args': [
+          '--num-retries=3',
+        ],
         'isolate_name': 'webkit_layout_tests_exparchive',
         'merge': {
           'args': [
@@ -1753,6 +1765,11 @@
           '--isolate-origins=http://www.web-platform.test:8001/,http://www1.web-platform.test:8001/,http://www2.web-platform.test:8001/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001/,http://xn--lve-6lad.web-platform.test:8001/,http://www.web-platform.test:8081/,http://www1.web-platform.test:8081/,http://www2.web-platform.test:8081/,http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8081/,http://xn--lve-6lad.web-platform.test:8081/,https://www.web-platform.test:8444/,https://www1.web-platform.test:8444/,https://www2.web-platform.test:8444/,https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8444/,https://xn--lve-6lad.web-platform.test:8444/',
           '--additional-expectations',
           'src/third_party/WebKit/LayoutTests/FlagExpectations/site-per-process',
+
+          # layout test failures are retried 3 times when '--test-list' is not
+          # passed, but 0 times when '--test-list' is passed. We want to always
+          # retry 3 times, so we explicitly specify it.
+          '--num-retries=3',
         ],
         'isolate_name': 'webkit_layout_tests_exparchive',
         'merge': {
@@ -1827,6 +1844,11 @@
       'webkit_layout_tests': {
         'args': [
           '--additional-driver-flag=--enable-features=NetworkService',
+
+          # layout test failures are retried 3 times when '--test-list' is not
+          # passed, but 0 times when '--test-list' is passed. We want to always
+          # retry 3 times, so we explicitly specify it.
+          '--num-retries=3',
         ],
         'isolate_name': 'webkit_layout_tests_exparchive',
         'merge': {
@@ -2212,6 +2234,12 @@
     # adding them to test_suite_exceptions.pyl.
     'webkit_layout_tests_isolated_scripts': {
       'webkit_layout_tests': {
+        # layout test failures are retried 3 times when '--test-list' is not
+        # passed, but 0 times when '--test-list' is passed. We want to always
+        # retry 3 times, so we explicitly specify it.
+        'args': [
+          '--num-retries=3',
+        ],
         'isolate_name': 'webkit_layout_tests_exparchive',
         'merge': {
           'args': [