[Benchmarking] Updating the story name checking logic in smoke test.

system_health_smoke_tests.py has a logic to make sure
we only run the latest version of a story in smoke test.

For example:
Case 1:
 Story_A:2019
 Story_A
Case 2:
 Story_A:2018
 Story_A:2019
In both cases, Story_A:2019 is expected to run, and all
other versions should be listed in the _DISABLED_TESTS in
order to save CQ run time.

However, in the current implementation, none of the old
versions in the above cases will be detected.This CL will
update the logic cover the above cases.

The new logic also discovered two stories which we currently
run smoke tests with their older versions. In this case, those
older versions now are added in the _DISABLED_TESTS. Those stories
are:
benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:social:vk
benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_desktop/load:social:vk:2018

benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile/load:social:twitter
benchmarks.system_health_smoke_test.SystemHealthBenchmarkSmokeTest.system_health.memory_mobile/load:social:twitter:2019


R=crouleau@google.com

Bug: catapult:1006771
Change-Id: Ia90f38568685ca8be45710f97e91cd8ea81dc856
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1845792
Commit-Queue: Wenbin Zhang <wenbinzhang@google.com>
Reviewed-by: Caleb Rouleau <crouleau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#704917}

tools/perf/benchmarks/smoketests_versions_validator_unittest.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for multiple version checking in system_health_smoke_test.py."""
+
+import unittest
+
+from benchmarks import system_health_smoke_test
+
+class TestSmoketestsVersionValidator(unittest.TestCase):
+  def test_version_check_pass(self):
+    all_stories = [
+        'a:2018', 'a', 'b:2018', 'b:2019', 'c:2018']
+    disabled_stories = frozenset([
+        'a:2018', 'b:2018'])
+    stories = system_health_smoke_test.find_multi_version_stories(
+        all_stories, disabled_stories)
+    self.assertEquals(0, len(stories),
+                      "Should be no result from version check")
+
+  def test_version_check_multi_colon_prefix_with_version(self):
+    all_stories = [
+        'story:name:a:2018', 'story:name:a:2019']
+    disabled_stories = frozenset(['story:name:a:2018'])
+    stories = system_health_smoke_test.find_multi_version_stories(
+        all_stories, disabled_stories)
+    self.assertEquals(0, len(stories),
+                      "Should be no result from version check")
+
+  def test_version_check_multi_colon_prefix_without_version(self):
+    all_stories = [
+        'story:name:a', 'story:name:a:2019']
+    disabled_stories = frozenset(['story:name:a:2019'])
+    stories = system_health_smoke_test.find_multi_version_stories(
+        all_stories, disabled_stories)
+    self.assertEquals(0, len(stories),
+                      "Should be no result from version check")
+
+  def test_version_check_fail_without_version(self):
+    all_stories = ['a', 'a:2019', 'b:2019']
+    disabled_stories = frozenset(['x'])
+    stories = system_health_smoke_test.find_multi_version_stories(
+        all_stories, disabled_stories)
+    self.assertEquals(1, len(stories),
+                      'Expecting 1 item in stories.')
+    self.assertIn('a', stories)
+    self.assertIn('a', stories['a'])
+    self.assertIn('a:2019', stories['a'])
+
+  def test_version_check_fail_with_version(self):
+    all_stories = ['a:2018', 'a:2019', 'b:2019']
+    disabled_stories = frozenset(['a'])
+    stories = system_health_smoke_test.find_multi_version_stories(
+        all_stories, disabled_stories)
+    self.assertEquals(1, len(stories),
+                      'Expecting 1 item in stories.')
+    self.assertIn('a', stories)
+    self.assertIn('a:2018', stories['a'])
+    self.assertIn('a:2019', stories['a'])
+
+  def test_version_check_fail_multiple_story(self):
+    all_stories = ['a', 'a:2019', 'b:2018', 'b:2019', 'c:2018']
+    disabled_stories = frozenset(['x'])
+    stories = system_health_smoke_test.find_multi_version_stories(
+        all_stories, disabled_stories)
+    self.assertEquals(2, len(stories),
+                      'Expecting 2 item in stories.')
+    self.assertIn('a', stories)
+    self.assertIn('a', stories['a'])
+    self.assertIn('a:2019', stories['a'])
+    self.assertIn('b', stories)
+    self.assertIn('b:2018', stories['b'])
+    self.assertIn('b:2019', stories['b'])
+
+if __name__ == '__main__':
+  unittest.main()

tools/perf/benchmarks/system_health_smoke_test.py

@@ -11,6 +11,8 @@ stories as memory ones, only with fewer actions (no memory dumping).
 
 import unittest
 
+from collections import defaultdict
+
 from core import path_util
 from core import perf_benchmark
 
@@ -75,6 +77,8 @@ _DISABLED_TESTS = frozenset({
   'system_health.memory_desktop/browse:news:reddit',
   'system_health.memory_desktop/browse:media:tumblr',
   'system_health.memory_desktop/browse:social:twitter_infinite_scroll',
+  'system_health.memory_mobile/load:social:twitter',
+  'system_health.memory_desktop/load:social:vk',
 
   # crbug.com/637230
   'system_health.memory_desktop/browse:news:cnn',
@@ -313,15 +317,54 @@ def load_tests(loader, standard_tests, pattern):
       names_stories_to_smoke_tests.append(
           benchmark_class.Name() + '/' + story_to_smoke_test.name)
 
-  for i, story_name in enumerate(names_stories_to_smoke_tests):
-    for j in xrange(i + 1, len(names_stories_to_smoke_tests)):
-      other_story_name = names_stories_to_smoke_tests[j]
-      if (other_story_name.startswith(story_name + ':') and
-          story_name not in _DISABLED_TESTS):
-        raise ValueError(
-            'Story %s is to be replaced by %s. Please put %s in '
-            '_DISABLED_TESTS list to save CQ capacity (see crbug.com/893615)). '
-            'You can use crbug.com/878390 for the disabling reference.' %
-            (repr(story_name), repr(other_story_name), repr(story_name)))
+  # The full story name should follow this convention: story_name[:version],
+  # where version is a year. Please refer to the link below for details:
+  # https://docs.google.com/document/d/134u_j_Lk2hLiDHYxK3NVdZM_sOtExrsExU-hiNFa1uw
+  # Raise exception for stories which have more than one version enabled.
+  multi_version_stories = find_multi_version_stories(
+      names_stories_to_smoke_tests, _DISABLED_TESTS)
+  if len(multi_version_stories):
+    msg = ''
+    for prefix, stories in multi_version_stories.items():
+      msg += prefix + ' : ' + ','.join(stories) + '\n'
+    raise ValueError(
+        'The stories below has multiple versions.'
+        'In order to save CQ capacity, we should only run the latest '
+        'version on CQ. Please put the legacy stories in _DISABLED_TESTS '
+        'list or remove them to save CQ capacity (see crbug.com/893615)). '
+        'You can use crbug.com/878390 for the disabling reference.'
+        '[StoryName] : [StoryVersion1],[StoryVersion2]...\n%s' % (msg))
 
   return suite
+
+
+def find_multi_version_stories(stories, disabled):
+  """Looks for stories with multiple versions enabled.
+
+  Args:
+    stories: list of strings, which are names of all the candidate stories.
+    disabled: frozenset of strings, which are names of stories which are
+      disabled.
+
+  Returns:
+    A dict mapping from a prefix string to a list of stories each of which
+    has the name with that prefix and has multiple versions enabled.
+  """
+  prefixes = defaultdict(list)
+  for name in stories:
+    if name in disabled:
+      continue
+    lastColon = name.rfind(':')
+    if lastColon == -1:
+      prefix = name
+    else:
+      version = name[lastColon+1:]
+      if version.isdigit():
+        prefix = name[:lastColon]
+      else:
+        prefix = name
+    prefixes[prefix].append(name)
+  for prefix, stories in prefixes.items():
+    if len(stories) == 1:
+      prefixes.pop(prefix)
+  return prefixes