Enable retry logic for all isolated script tests

This CL enables logic to only retry failed tests for most isolated script tests. The following test suites are affected: * angle_perftests * chromedriver_py_tests * components_perftests * content_shell_crash_test * devtools_closure_compile * devtools_eslint * heap_profiling.mobile.disabled * heap_profiling.mobile.native * heap_profiling.mobile.pseudo * memory.leak_detection * metrics_python_tests * mini_installer_tests * monochrome_apk_checker * passthrough_command_buffer_perftests * performance_test_suite * site_per_process_webkit_layout_tests * telemetry_gpu_unittests * telemetry_perf_unittests * telemetry_perf_unittests_viz * telemetry_unittests * telemetry_unittests_viz * validating_command_buffer_perftests * views_perftests * webkit_python_tests These should all work with this logic; I've tested these manually myself. NOTRY=true NOPRESUBMIT=true Bug: 533481 Change-Id: If3a5a03410fd7f1f597197d98ae3060bd91ff487 Reviewed-on: https://chromium-review.googlesource.com/1141340 Commit-Queue: Stephen Martinis <martiniss@chromium.org> Reviewed-by: John Budorick <jbudorick@chromium.org> Reviewed-by: Kenneth Russell <kbr@chromium.org> Reviewed-by: Dirk Pranke <dpranke@chromium.org> Cr-Commit-Position: refs/heads/master@{#577315}

Enable retry logic for all isolated script tests
This CL enables logic to only retry failed tests for most isolated script tests. The following test suites are affected: * angle_perftests * chromedriver_py_tests * components_perftests * content_shell_crash_test * devtools_closure_compile * devtools_eslint * heap_profiling.mobile.disabled * heap_profiling.mobile.native * heap_profiling.mobile.pseudo * memory.leak_detection * metrics_python_tests * mini_installer_tests * monochrome_apk_checker * passthrough_command_buffer_perftests * performance_test_suite * site_per_process_webkit_layout_tests * telemetry_gpu_unittests * telemetry_perf_unittests * telemetry_perf_unittests_viz * telemetry_unittests * telemetry_unittests_viz * validating_command_buffer_perftests * views_perftests * webkit_python_tests These should all work with this logic; I've tested these manually myself. NOTRY=true NOPRESUBMIT=true Bug: 533481 Change-Id: If3a5a03410fd7f1f597197d98ae3060bd91ff487 Reviewed-on: https://chromium-review.googlesource.com/1141340 Commit-Queue: Stephen Martinis <martiniss@chromium.org> Reviewed-by: John Budorick <jbudorick@chromium.org> Reviewed-by: Kenneth Russell <kbr@chromium.org> Reviewed-by: Dirk Pranke <dpranke@chromium.org> Cr-Commit-Position: refs/heads/master@{#577315}
65e7fd98 · Stephen Martinis · Commit Bot · a87d204e · 65e7fd98 · 65e7fd98
Commit 65e7fd98 authored Jul 23, 2018 by Stephen Martinis Committed by Commit Bot Jul 23, 2018
15 changed files
--- a/testing/buildbot/chromium.android.fyi.json
+++ b/testing/buildbot/chromium.android.fyi.json
@@ -601,6 +601,7 @@
          "script": "//tools/perf/process_perf_results.py"
        },
        "name": "heap_profiling.mobile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -6336,6 +6337,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [

--- a/testing/buildbot/chromium.android.json
+++ b/testing/buildbot/chromium.android.json
@@ -19148,6 +19148,7 @@
      {
        "isolate_name": "monochrome_apk_checker",
        "name": "monochrome_apk_checker",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -21818,6 +21819,7 @@
      {
        "isolate_name": "components_perftests",
        "name": "components_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -21832,6 +21834,7 @@
      {
        "isolate_name": "monochrome_apk_checker",
        "name": "monochrome_apk_checker",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -21850,6 +21853,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [

--- a/testing/buildbot/chromium.clang.json
+++ b/testing/buildbot/chromium.clang.json
@@ -10133,6 +10133,7 @@
      {
        "isolate_name": "monochrome_apk_checker",
        "name": "monochrome_apk_checker",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -16749,6 +16750,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -16756,6 +16758,7 @@
      {
        "isolate_name": "components_perftests",
        "name": "components_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -16763,6 +16766,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -16770,6 +16774,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -16777,6 +16782,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -16784,6 +16790,7 @@
      {
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -16796,6 +16803,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -16804,6 +16812,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -16834,6 +16843,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }

--- a/testing/buildbot/chromium.fyi.json
+++ b/testing/buildbot/chromium.fyi.json
@@ -813,6 +813,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -820,6 +821,7 @@
      {
        "isolate_name": "components_perftests",
        "name": "components_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -827,6 +829,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -834,6 +837,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -841,6 +845,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -848,6 +853,7 @@
      {
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -860,6 +866,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -868,6 +875,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -891,6 +899,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1046,6 +1055,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -1062,6 +1072,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests_viz",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -1071,6 +1082,7 @@
      {
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -1084,6 +1096,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests_viz",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -2971,6 +2984,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -2996,6 +3010,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4717,6 +4732,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4724,6 +4740,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4731,6 +4748,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4738,6 +4756,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4745,6 +4764,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4766,6 +4786,7 @@
          "script": "//third_party/blink/tools/merge_web_test_results.py"
        },
        "name": "site_per_process_webkit_layout_tests",
+        "only_retry_failed_tests": true,
        "results_handler": "layout tests",
        "swarming": {
          "can_use_on_swarming_builders": true,
@@ -4780,6 +4801,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4787,6 +4809,7 @@
      {
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -4799,6 +4822,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -4807,6 +4831,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4830,6 +4855,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }

--- a/testing/buildbot/chromium.gpu.fyi.json
+++ b/testing/buildbot/chromium.gpu.fyi.json
--- a/testing/buildbot/chromium.gpu.json
+++ b/testing/buildbot/chromium.gpu.json
--- a/testing/buildbot/chromium.linux.json
+++ b/testing/buildbot/chromium.linux.json
@@ -792,6 +792,7 @@
          "script": "//tools/perf/process_perf_results.py"
        },
        "name": "memory.leak_detection",
+        "only_retry_failed_tests": true,
        "override_compile_targets": [
          "performance_test_suite"
        ],
@@ -1529,6 +1530,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1536,6 +1538,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1543,6 +1546,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1550,6 +1554,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1557,6 +1562,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1578,6 +1584,7 @@
          "script": "//third_party/blink/tools/merge_web_test_results.py"
        },
        "name": "site_per_process_webkit_layout_tests",
+        "only_retry_failed_tests": true,
        "results_handler": "layout tests",
        "swarming": {
          "can_use_on_swarming_builders": true,
@@ -1592,6 +1599,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1603,6 +1611,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -1615,6 +1624,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -1623,6 +1633,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1651,6 +1662,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2371,6 +2383,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2378,6 +2391,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2385,6 +2399,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2392,6 +2407,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2399,6 +2415,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2410,6 +2427,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -2422,6 +2440,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -2430,6 +2449,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2461,6 +2481,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3059,6 +3080,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3066,6 +3088,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3073,6 +3096,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3080,6 +3104,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3090,6 +3115,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -3098,6 +3124,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3105,6 +3132,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -4349,6 +4377,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4361,6 +4390,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4373,6 +4403,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4385,6 +4416,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4397,6 +4429,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4423,6 +4456,7 @@
          "script": "//third_party/blink/tools/merge_web_test_results.py"
        },
        "name": "site_per_process_webkit_layout_tests",
+        "only_retry_failed_tests": true,
        "results_handler": "layout tests",
        "swarming": {
          "can_use_on_swarming_builders": true,
@@ -4437,6 +4471,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4453,6 +4488,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4470,6 +4506,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4483,6 +4520,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -4516,6 +4554,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [

--- a/testing/buildbot/chromium.mac.json
+++ b/testing/buildbot/chromium.mac.json
--- a/testing/buildbot/chromium.swarm.json
+++ b/testing/buildbot/chromium.swarm.json
@@ -696,6 +696,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -716,6 +717,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [

--- a/testing/buildbot/chromium.win.json
+++ b/testing/buildbot/chromium.win.json
@@ -652,6 +652,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -659,6 +660,7 @@
      {
        "isolate_name": "components_perftests",
        "name": "components_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -666,6 +668,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -673,6 +676,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -680,6 +684,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -687,6 +692,7 @@
      {
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -699,6 +705,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -707,6 +714,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -714,6 +722,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -1379,6 +1388,7 @@
      {
        "isolate_name": "mini_installer_tests",
        "name": "mini_installer_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [
@@ -1997,6 +2007,7 @@
        "experiment_percentage": 100,
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2004,6 +2015,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2023,6 +2035,7 @@
          "script": "//tools/perf/process_perf_results.py"
        },
        "name": "performance_test_suite",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -2032,6 +2045,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2042,6 +2056,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -2050,6 +2065,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2057,6 +2073,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2711,6 +2728,7 @@
        ],
        "isolate_name": "chromedriver_py_tests",
        "name": "chromedriver_py_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2718,6 +2736,7 @@
      {
        "isolate_name": "components_perftests",
        "name": "components_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2725,6 +2744,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2732,6 +2752,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2739,6 +2760,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2746,6 +2768,7 @@
      {
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -2758,6 +2781,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -2766,6 +2790,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -2789,6 +2814,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3469,6 +3495,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3476,6 +3503,7 @@
      {
        "isolate_name": "metrics_python_tests",
        "name": "metrics_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3494,6 +3522,7 @@
          "script": "//tools/perf/process_perf_results.py"
        },
        "name": "performance_test_suite",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -3503,6 +3532,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3514,6 +3544,7 @@
        "experiment_percentage": 100,
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -3526,6 +3557,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -3534,6 +3566,7 @@
      {
        "isolate_name": "views_perftests",
        "name": "views_perftests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -3568,6 +3601,7 @@
      {
        "isolate_name": "webkit_python_tests",
        "name": "webkit_python_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }

--- a/testing/buildbot/client.v8.chromium.json
+++ b/testing/buildbot/client.v8.chromium.json
@@ -225,6 +225,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -232,6 +233,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -239,6 +241,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -246,6 +249,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -256,6 +260,7 @@
        ],
        "isolate_name": "telemetry_perf_unittests",
        "name": "telemetry_perf_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "hard_timeout": 960,
@@ -268,6 +273,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4
@@ -500,6 +506,7 @@
      {
        "isolate_name": "content_shell_crash_test",
        "name": "content_shell_crash_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -507,6 +514,7 @@
      {
        "isolate_name": "devtools_closure_compile",
        "name": "devtools_closure_compile",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -514,6 +522,7 @@
      {
        "isolate_name": "devtools_eslint",
        "name": "devtools_eslint",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -521,6 +530,7 @@
      {
        "isolate_name": "telemetry_gpu_unittests",
        "name": "telemetry_gpu_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -531,6 +541,7 @@
        ],
        "isolate_name": "telemetry_unittests",
        "name": "telemetry_unittests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "shards": 4

--- a/testing/buildbot/client.v8.fyi.json
+++ b/testing/buildbot/client.v8.fyi.json
--- a/testing/buildbot/generate_buildbot_json.py
+++ b/testing/buildbot/generate_buildbot_json.py
@@ -500,6 +500,9 @@ class BBJSONGenerator(object):
    self.initialize_args_for_test(result, tester_config)
    result = self.update_and_cleanup_test(result, test_name, tester_name)
    self.add_common_test_properties(result, tester_config)
+    # TODO(martiniss): Remove this. https://crbug.com/533481
+    result['only_retry_failed_tests'] = True
    return result
  def generate_script_test(self, waterfall, tester_name, tester_config,

--- a/testing/buildbot/generate_buildbot_json_unittest.py
+++ b/testing/buildbot/generate_buildbot_json_unittest.py
@@ -804,6 +804,7 @@ ISOLATED_SCRIPT_OUTPUT = """\
      {
        "isolate_name": "foo_test",
        "name": "foo_test",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true
        }
@@ -925,6 +926,7 @@ GPU_TELEMETRY_TEST_OUTPUT = """\
        ],
        "isolate_name": "telemetry_gpu_integration_test",
        "name": "foo_tests",
+        "only_retry_failed_tests": true,
        "swarming": {
          "can_use_on_swarming_builders": true,
          "dimension_sets": [

--- a/testing/buildbot/test_suites.pyl
+++ b/testing/buildbot/test_suites.pyl
@@ -614,7 +614,6 @@
        ],
        'script': '//third_party/blink/tools/merge_web_test_results.py',
      },
-      'only_retry_failed_tests': True,
      'results_handler': 'layout tests',
      'swarming': {
        'shards': 12,
@@ -873,7 +872,6 @@
        ],
        'script': '//third_party/blink/tools/merge_web_test_results.py',
      },
-      'only_retry_failed_tests': True,
      'results_handler': 'layout tests',
      'swarming': {
        'shards': 12,
@@ -1677,7 +1675,6 @@
        ],
        'script': '//third_party/blink/tools/merge_web_test_results.py',
      },
-      'only_retry_failed_tests': True,
      'results_handler': 'layout tests',
      'swarming': {
        'shards': 12,
@@ -2124,7 +2121,6 @@
        ],
        'script': '//third_party/blink/tools/merge_web_test_results.py',
      },
-      'only_retry_failed_tests': True,
      'results_handler': 'layout tests',
    },
  },