Ensure gtest benchmark's name is the same as its corresponding test step's name

Bug:870692,757933
Cq-Include-Trybots: master.tryserver.chromium.perf:obbs_fyi
Change-Id: Ifc70a71cf4ddb2628038b45b51a57381a2a41c3a

NOTRY=true  # CQ flake

Change-Id: Ifc70a71cf4ddb2628038b45b51a57381a2a41c3a
Reviewed-on: https://chromium-review.googlesource.com/1162584
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Reviewed-by: David Tu <dtu@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Cr-Commit-Position: refs/heads/master@{#580724}

testing/buildbot/chromium.perf.fyi.json

@@ -141,6 +141,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "net_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -226,6 +228,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "views_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -271,6 +275,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "load_library_perf_tests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],

testing/buildbot/chromium.perf.json

@@ -11,6 +11,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "angle_perftests",
           "--non-telemetry=true",
           "--migrated-test=true",
           "--shard-timeout=300"
@@ -55,6 +57,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "components_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -98,6 +102,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "gpu_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -188,6 +194,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "tracing_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -384,6 +392,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "tracing_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -447,6 +457,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "angle_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -488,6 +500,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "load_library_perf_tests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -529,6 +543,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "media_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -570,6 +586,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "passthrough_command_buffer_perftests",
           "--non-telemetry=true",
           "--migrated-test=true",
           "--use-cmd-decoder=passthrough",
@@ -658,6 +676,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "validating_command_buffer_perftests",
           "--non-telemetry=true",
           "--migrated-test=true",
           "--use-cmd-decoder=validating",
@@ -705,6 +725,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "components_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -746,6 +768,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "load_library_perf_tests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -787,6 +811,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "media_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -936,6 +962,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "angle_perftests",
           "--non-telemetry=true",
           "--migrated-test=true",
           "--shard-timeout=300"
@@ -980,6 +1008,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "components_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1023,6 +1053,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "gpu_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1066,6 +1098,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "media_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1156,6 +1190,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "tracing_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1203,6 +1239,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "load_library_perf_tests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1244,6 +1282,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "media_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1285,6 +1325,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "net_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1371,6 +1413,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "tracing_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1416,6 +1460,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "load_library_perf_tests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1457,6 +1503,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "performance_browser_tests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1547,6 +1595,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "media_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1588,6 +1638,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "net_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1674,6 +1726,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "views_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1719,6 +1773,8 @@
     "isolated_scripts": [
       {
         "args": [
+          "--gtest-benchmark-name",
+          "components_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1760,6 +1816,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "media_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],
@@ -1846,6 +1904,8 @@
       },
       {
         "args": [
+          "--gtest-benchmark-name",
+          "views_perftests",
           "--non-telemetry=true",
           "--migrated-test=true"
         ],

testing/scripts/run_performance_tests.py

@@ -178,6 +178,10 @@ def main():
   parser.add_argument('--xvfb', help='Start xvfb.', action='store_true')
   parser.add_argument('--non-telemetry',
                       help='Type of perf test', type=bool, default=False)
+  parser.add_argument('--gtest-benchmark-name',
+                      help='Name of the gtest benchmark', type=str,
+                      required=False)
+
   parser.add_argument('--benchmarks',
                       help='Comma separated list of benchmark names'
                       ' to run in lieu of indexing into our benchmark bot maps',
@@ -194,8 +198,13 @@ def main():
   return_code = 0
 
   if args.non_telemetry:
-    # For non telemetry tests the benchmark name is the name of the executable.
-    benchmark_name = rest_args[0]
+    benchmark_name = args.gtest_benchmark_name
+    # Fallback to use the name of the executable if flag isn't set.
+    # TODO(crbug.com/870899): remove fallback logic and raise parser error if
+    # -non-telemetry is set but --gtest-benchmark-name is not set once pinpoint
+    # is converted to always pass --gtest-benchmark-name flag.
+    if not benchmark_name:
+      benchmark_name = rest_args[0]
     return_code, charts, output_json = run_gtest_perf_test.execute_perf_test(
         args, rest_args)
 

tools/perf/core/perf_data_generator.py

@@ -902,23 +902,21 @@ def generate_telemetry_args(tester_config):
 
   return test_args
 
-def generate_non_telemetry_args():
+def generate_non_telemetry_args(test_name):
+  # --gtest-benchmark-name so the benchmark name is consistent with the test
+  # step's name. This is not always the same as the test binary's name (see
+  # crbug.com/870692).
   # --non-telemetry tells run_performance_tests.py that this test needs
   #   to be executed differently
   # --migrated-test tells run_performance_test_wrapper that this has
   #   non-telemetry test has been migrated to the new recipe.
   return [
+    '--gtest-benchmark-name', test_name,
     '--non-telemetry=true',
     '--migrated-test=true'
   ]
 
 def generate_performance_test(tester_config, test):
-  if test.get('telemetry', True):
-    test_args = generate_telemetry_args(tester_config)
-  else:
-    test_args = generate_non_telemetry_args()
-  # Append any additional args specific to an isolate
-  test_args += test.get('extra_args', [])
   isolate_name = test['isolate']
 
   # Check to see if the name is different than the isolate
@@ -926,6 +924,13 @@ def generate_performance_test(tester_config, test):
   if test.get('test_suite', False):
     test_suite = test['test_suite']
 
+  if test.get('telemetry', True):
+    test_args = generate_telemetry_args(tester_config)
+  else:
+    test_args = generate_non_telemetry_args(test_name=test_suite)
+  # Append any additional args specific to an isolate
+  test_args += test.get('extra_args', [])
+
   result = {
     'args': test_args,
     'isolate_name': isolate_name,

tools/perf/core/perf_data_generator_unittest.py

@@ -104,7 +104,8 @@ class PerfDataGeneratorTest(unittest.TestCase):
     expected_generated_test = {
         'override_compile_targets': ['angle_perftest'],
         'isolate_name': 'angle_perftest',
-        'args': ['--non-telemetry=true', '--migrated-test=true'],
+        'args': ['--gtest-benchmark-name', 'angle_perftest',
+                 '--non-telemetry=true', '--migrated-test=true'],
         'trigger_script': {
           'args': [
             '--multiple-dimension-script-verbose',