Always put Telemetry tests at the end to avoid starving fast tests

This also add this check to PRESUBMIT perf_json_config_validator script

Bug: 873389
Cq-Include-Trybots: master.tryserver.chromium.perf:obbs_fyi
Change-Id: I16fd762e567d5a41d75a221456e12e14a03f0b85
Reviewed-on: https://chromium-review.googlesource.com/1171578Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#582422}

testing/buildbot/chromium.perf.fyi.json

@@ -184,13 +184,12 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=release",
-          "--upload-results",
-          "--run-ref-build",
-          "--test-shard-map-filename=mac1012_5_shard_map.json"
+          "--gtest-benchmark-name",
+          "views_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "views_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -198,9 +197,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "views_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "views_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -215,7 +214,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 5,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -228,12 +227,13 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "views_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true"
+          "-v",
+          "--browser=release",
+          "--upload-results",
+          "--run-ref-build",
+          "--test-shard-map-filename=mac1012_5_shard_map.json"
         ],
-        "isolate_name": "views_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -241,9 +241,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "views_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "views_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -258,7 +258,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 5,
           "upload_test_results": true
         },
         "trigger_script": {

testing/buildbot/chromium.perf.json

@@ -147,14 +147,12 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=android-chromium",
-          "--upload-results",
-          "--run-ref-build",
-          "--test-shard-map-filename=android_nexus5_16_shard_map.json",
-          "--assert-gpu-compositing"
+          "--gtest-benchmark-name",
+          "tracing_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "tracing_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -162,9 +160,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "tracing_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "tracing_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -181,7 +179,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 16,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -194,12 +192,14 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "tracing_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true"
+          "-v",
+          "--browser=android-chromium",
+          "--upload-results",
+          "--run-ref-build",
+          "--test-shard-map-filename=android_nexus5_16_shard_map.json",
+          "--assert-gpu-compositing"
         ],
-        "isolate_name": "tracing_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -207,9 +207,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "tracing_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "tracing_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -226,7 +226,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 16,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -535,14 +535,14 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=release_x64",
-          "--upload-results",
-          "--run-ref-build",
-          "--test-shard-map-filename=win7_nvidia_shard_map.json",
-          "--assert-gpu-compositing"
+          "--gtest-benchmark-name",
+          "validating_command_buffer_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true",
+          "--use-cmd-decoder=validating",
+          "--use-stub"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "command_buffer_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -550,9 +550,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "validating_command_buffer_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "command_buffer_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -567,7 +567,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 5,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -580,14 +580,14 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "validating_command_buffer_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true",
-          "--use-cmd-decoder=validating",
-          "--use-stub"
+          "-v",
+          "--browser=release_x64",
+          "--upload-results",
+          "--run-ref-build",
+          "--test-shard-map-filename=win7_nvidia_shard_map.json",
+          "--assert-gpu-compositing"
         ],
-        "isolate_name": "command_buffer_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -595,9 +595,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "validating_command_buffer_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "command_buffer_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -612,7 +612,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 5,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1047,14 +1047,12 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=android-chromium",
-          "--upload-results",
-          "--run-ref-build",
-          "--test-shard-map-filename=android_nexus5x_16_shard_map.json",
-          "--assert-gpu-compositing"
+          "--gtest-benchmark-name",
+          "tracing_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "tracing_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1062,9 +1060,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "tracing_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "tracing_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1081,7 +1079,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 16,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1094,12 +1092,14 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "tracing_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true"
+          "-v",
+          "--browser=android-chromium",
+          "--upload-results",
+          "--run-ref-build",
+          "--test-shard-map-filename=android_nexus5x_16_shard_map.json",
+          "--assert-gpu-compositing"
         ],
-        "isolate_name": "tracing_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1107,9 +1107,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "tracing_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "tracing_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1126,7 +1126,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 16,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1272,14 +1272,12 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=release",
-          "--upload-results",
-          "--run-ref-build",
-          "--test-shard-map-filename=linux_perf_shard_map.json",
-          "--assert-gpu-compositing"
+          "--gtest-benchmark-name",
+          "tracing_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "tracing_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1287,9 +1285,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "tracing_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "tracing_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1304,7 +1302,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 26,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1317,12 +1315,14 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "tracing_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true"
+          "-v",
+          "--browser=release",
+          "--upload-results",
+          "--run-ref-build",
+          "--test-shard-map-filename=linux_perf_shard_map.json",
+          "--assert-gpu-compositing"
         ],
-        "isolate_name": "tracing_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1330,9 +1330,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "tracing_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "tracing_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1347,7 +1347,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 26,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1584,13 +1584,12 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=release",
-          "--upload-results",
-          "--test-shard-map-filename=mac_1013_high_end_26_shard_map.json",
-          "--assert-gpu-compositing"
+          "--gtest-benchmark-name",
+          "views_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "views_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1598,9 +1597,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "views_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "views_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1615,7 +1614,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 26,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1628,12 +1627,13 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "views_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true"
+          "-v",
+          "--browser=release",
+          "--upload-results",
+          "--test-shard-map-filename=mac_1013_high_end_26_shard_map.json",
+          "--assert-gpu-compositing"
         ],
-        "isolate_name": "views_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1641,9 +1641,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "views_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "views_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1658,7 +1658,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 26,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1761,14 +1761,12 @@
       },
       {
         "args": [
-          "-v",
-          "--browser=release_x64",
-          "--upload-results",
-          "--run-ref-build",
-          "--test-shard-map-filename=win10_shard_map.json",
-          "--assert-gpu-compositing"
+          "--gtest-benchmark-name",
+          "views_perftests",
+          "--non-telemetry=true",
+          "--migrated-test=true"
         ],
-        "isolate_name": "performance_test_suite",
+        "isolate_name": "views_perftests",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1776,9 +1774,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "performance_test_suite",
+        "name": "views_perftests",
         "override_compile_targets": [
-          "performance_test_suite"
+          "views_perftests"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1793,7 +1791,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 26,
+          "shards": 1,
           "upload_test_results": true
         },
         "trigger_script": {
@@ -1806,12 +1804,14 @@
       },
       {
         "args": [
-          "--gtest-benchmark-name",
-          "views_perftests",
-          "--non-telemetry=true",
-          "--migrated-test=true"
+          "-v",
+          "--browser=release_x64",
+          "--upload-results",
+          "--run-ref-build",
+          "--test-shard-map-filename=win10_shard_map.json",
+          "--assert-gpu-compositing"
         ],
-        "isolate_name": "views_perftests",
+        "isolate_name": "performance_test_suite",
         "merge": {
           "args": [
             "--service-account-file",
@@ -1819,9 +1819,9 @@
           ],
           "script": "//tools/perf/process_perf_results.py"
         },
-        "name": "views_perftests",
+        "name": "performance_test_suite",
         "override_compile_targets": [
-          "views_perftests"
+          "performance_test_suite"
         ],
         "swarming": {
           "can_use_on_swarming_builders": true,
@@ -1836,7 +1836,7 @@
           "hard_timeout": 25200,
           "ignore_task_failure": false,
           "io_timeout": 1800,
-          "shards": 1,
+          "shards": 26,
           "upload_test_results": true
         },
         "trigger_script": {

tools/perf/core/perf_data_generator.py

@@ -70,14 +70,6 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
   'testers' : {
     'OBBS Mac 10.12 Perf': {
       'tests': [
-        {
-          'isolate': 'performance_test_suite',
-          'extra_args': [
-            '--run-ref-build',
-            '--test-shard-map-filename=mac1012_5_shard_map.json',
-          ],
-          'num_shards': 5
-        },
         {
           'isolate': 'net_perftests',
           'num_shards': 1,
@@ -87,6 +79,14 @@ NEW_PERF_RECIPE_FYI_TESTERS = {
           'isolate': 'views_perftests',
           'num_shards': 1,
           'telemetry': False,
+        },
+        {
+          'isolate': 'performance_test_suite',
+          'extra_args': [
+            '--run-ref-build',
+            '--test-shard-map-filename=mac1012_5_shard_map.json',
+          ],
+          'num_shards': 5
         }
       ],
       'platform': 'mac',
@@ -949,11 +949,20 @@ def load_and_update_fyi_json(fyi_waterfall_file):
 
 def generate_telemetry_tests(testers, tests):
   for tester, tester_config in testers['testers'].iteritems():
-    isolated_scripts = []
+    telemetry_tests = []
+    gtest_tests = []
     for test in tester_config['tests']:
-      isolated_scripts.append(generate_performance_test(tester_config, test))
+      generated_script = generate_performance_test(tester_config, test)
+      if test.get('telemetry', True):
+        telemetry_tests.append(generated_script)
+      else:
+        gtest_tests.append(generated_script)
+    telemetry_tests.sort(key=lambda x: x['name'])
+    gtest_tests.sort(key=lambda x: x['name'])
     tests[tester] = {
-      'isolated_scripts': sorted(isolated_scripts, key=lambda x: x['name'])
+      # Put Telemetry tests as the end since they tend to run longer to avoid
+      # starving gtests (see crbug.com/873389).
+      'isolated_scripts': gtest_tests + telemetry_tests
     }
 
 

tools/perf/core/perf_json_config_validator.py

@@ -101,7 +101,9 @@ def _ValidateBrowserType(builder_name, test_config):
 
 def ValidateTestingBuilder(builder_name, builder_data):
   isolated_scripts = builder_data['isolated_scripts']
+  test_names = []
   for test_config in isolated_scripts:
+    test_names.append(test_config['name'])
     _ValidateSwarmingDimension(
         builder_name,
         swarming_dimensions=test_config['swarming'].get('dimension_sets', {}))
@@ -110,6 +112,17 @@ def ValidateTestingBuilder(builder_name, builder_data):
       _ValidateShardingData(builder_name, test_config)
       _ValidateBrowserType(builder_name, test_config)
 
+  if ('performance_test_suite' in test_names or
+      'performance_webview_test_suite' in test_names):
+    if test_names[-1] not in ('performance_test_suite',
+                              'performance_webview_test_suite'):
+      raise ValueError(
+          'performance_test_suite or performance_webview_test_suite must run '
+          'at the end of builder %s to avoid starving other test step '
+          '(see crbug.com/873389). Instead found %s' % (
+            repr(builder_name), test_names[-1]))
+
+
 
 def _IsBuilderName(name):
   return not name.startswith('AAA')