Disable smoke testing of legacy system health stories which already have 2018 version

This also add check to ensure that legacy system health stories which already
have a newer version must be disabled within smoke testing.

Finally, this CL also shorten the names used in _DISABLED to simplify the job of
disabling a system health story.

Bug: 878390, 893615
Change-Id: Ife447c9e5935c960ea7e1818adfcaef0ccaeb59a
Reviewed-on: https://chromium-review.googlesource.com/c/1289686Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#600987}

tools/perf/benchmarks/system_health_smoke_test.py

@@ -34,44 +34,68 @@ def GetSystemHealthBenchmarksToSmokeTest():
 
 
 _DISABLED_TESTS = frozenset({
-  # crbug.com/733427
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile/browse:news:cnn',  # pylint: disable=line-too-long
+  # crbug.com/878390 - These stories are already covered by their 2018 versions
+  # and will later be removed.
+  'system_health.memory_mobile/browse:tech:discourse_infinite_scroll',
+  'system_health.memory_mobile/browse:social:facebook_infinite_scroll',
+  'system_health.memory_mobile/browse:news:cnn',
+  'system_health.memory_mobile/load:news:cnn',
+  'system_health.memory_mobile/load:tools:stackoverflow',
+  'system_health.memory_desktop/load_accessibility:shopping:amazon',
+  'system_health.memory_desktop/browse_accessibility:tech:codesearch',
+  'system_health.memory_desktop/load_accessibility:media:wikipedia',
+  'system_health.memory_desktop/browse:tech:discourse_infinite_scroll',
+  'system_health.memory_desktop/browse:social:facebook_infinite_scroll',
+  'system_health.memory_desktop/browse:news:flipboard',
+  'system_health.memory_desktop/browse:search:google',
+  'system_health.memory_desktop/browse:news:hackernews',
+  'system_health.memory_desktop/load:search:amazon',
+  'system_health.memory_desktop/load:news:bbc',
+  'system_health.memory_desktop/load:news:hackernews',
+  'system_health.memory_desktop/load:social:instagram',
+  'system_health.memory_desktop/load:news:reddit',
+  'system_health.memory_desktop/load:search:taobao',
+  'system_health.memory_desktop/multitab:misc:typical24',
+  'system_health.memory_desktop/browse:news:reddit',
+  'system_health.memory_desktop/browse:media:tumblr',
+  'system_health.memory_desktop/browse:social:twitter_infinite_scroll',
+
   # crbug.com/637230
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/browse:news:cnn',  # pylint: disable=line-too-long
+  'system_health.memory_desktop/browse:news:cnn',
   # Permenently disabled from smoke test for being long-running.
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile/long_running:tools:gmail-foreground',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile/long_running:tools:gmail-background',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/long_running:tools:gmail-foreground',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/long_running:tools:gmail-background',  # pylint: disable=line-too-long
+  'system_health.memory_mobile/long_running:tools:gmail-foreground',
+  'system_health.memory_mobile/long_running:tools:gmail-background',
+  'system_health.memory_desktop/long_running:tools:gmail-foreground',
+  'system_health.memory_desktop/long_running:tools:gmail-background',
 
   # crbug.com/769263
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/play:media:soundcloud',  # pylint: disable=line-too-long
+  'system_health.memory_desktop/play:media:soundcloud',
 
   # crbug.com/
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/browse:news:nytimes',  # pylint: disable=line-too-long
+  'system_health.memory_desktop/browse:news:nytimes',
 
   # crbug.com/688190
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile/browse:news:washingtonpost',  # pylint: disable=line-too-long
+  'system_health.memory_mobile/browse:news:washingtonpost',
 
   # crbug.com/696824
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:news:qq',  # pylint: disable=line-too-long
+  'system_health.memory_desktop/load:news:qq',
   # crbug.com/893615
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:news:cnn',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:tools:stackoverflow',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:games:alphabetty',  # pylint: disable=line-too-long
+  'system_health.memory_desktop/load:news:cnn',
+  'system_health.memory_desktop/load:tools:stackoverflow',
+  'system_health.memory_desktop/load:games:alphabetty',
 
   # crbug.com/698006
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:tools:drive',  # pylint: disable=line-too-long
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:tools:gmail',  # pylint: disable=line-too-long
+  'system_health.memory_desktop/load:tools:drive',
+  'system_health.memory_desktop/load:tools:gmail',
 
   # crbug.com/725386
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/browse:social:twitter', # pylint: disable=line-too-long
+  'system_health.memory_desktop/browse:social:twitter',
 
   # crbug.com/816482
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:news:nytimes', # pylint: disable=line-too-long
+  'system_health.memory_desktop/load:news:nytimes',
 
   # crbug.com/885320
-  'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/browse:search:google:2018', # pylint: disable=line-too-long
+  'system_health.memory_desktop/browse:search:google:2018',
 })
 
 
@@ -117,7 +141,16 @@ def _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test):
     if possible_browser is None:
       self.skipTest('Cannot find the browser to run the test.')
 
-    if self.id() in _DISABLED_TESTS and not options.run_disabled_tests:
+
+    simplified_test_name = self.id().replace(
+        'benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.',
+        '')
+
+    # Sanity check to ensure that that substring removal was effective.
+    assert len(simplified_test_name) < len(self.id())
+
+    if (simplified_test_name in _DISABLED_TESTS and
+        not options.run_disabled_tests):
       self.skipTest('Test is explicitly disabled')
 
     single_page_benchmark = SinglePageBenchmark()
@@ -172,6 +205,7 @@ def load_tests(loader, standard_tests, pattern):
   suite = progress_reporter.TestSuite()
   benchmark_classes = GetSystemHealthBenchmarksToSmokeTest()
   assert benchmark_classes, 'This list should never be empty'
+  names_stories_to_smoke_tests = []
   for benchmark_class in benchmark_classes:
 
     # HACK: these options should be derived from options_for_unittests which are
@@ -193,4 +227,18 @@ def load_tests(loader, standard_tests, pattern):
       suite.addTest(
           _GenerateSmokeTestCase(benchmark_class, story_to_smoke_test))
 
+      names_stories_to_smoke_tests.append(
+          benchmark_class.Name() + '/' + story_to_smoke_test.name)
+
+  for i, story_name in enumerate(names_stories_to_smoke_tests):
+    for j in xrange(i + 1, len(names_stories_to_smoke_tests)):
+      other_story_name = names_stories_to_smoke_tests[j]
+      if (other_story_name.startswith(story_name + ':') and
+          story_name not in _DISABLED_TESTS):
+        raise ValueError(
+            'Story %s is to be replaced by %s. Please put %s in '
+            '_DISABLED_TESTS list to save CQ capacity (see crbug.com/893615)). '
+            'You can use crbug.com/878390 for the disabling reference.' %
+            (repr(story_name), repr(other_story_name), repr(story_name)))
+
   return suite