[Telemetry] Use new expectations file when validating disabled stories names.

Bug: 781409
Change-Id: I89a3561584e18dc56e411bceb0ee25d72e996989
Reviewed-on: https://chromium-review.googlesource.com/818331Reviewed-by: Charlie Andrews <charliea@chromium.org>
Reviewed-by: Ned Nguyen <nednguyen@google.com>
Commit-Queue: rnephew <rnephew@chromium.org>
Cr-Commit-Position: refs/heads/master@{#524090}

tools/perf/core/story_expectation_validator.py

@@ -38,11 +38,12 @@ def check_decorators(benchmarks):
           'information. \nBenchmark: %s' % benchmark.Name())
 
 
-def validate_story_names(benchmarks):
+def validate_story_names(benchmarks, raw_expectations_data):
   for benchmark in benchmarks:
     if benchmark.Name() in CLUSTER_TELEMETRY_BENCHMARKS:
       continue
     b = benchmark()
+    b.AugmentExpectationsWithParser(raw_expectations_data)
     options = browser_options.BrowserFinderOptions()
     # tabset_repeat is needed for tab_switcher benchmarks.
     options.tabset_repeat = 1
@@ -55,7 +56,7 @@ def validate_story_names(benchmarks):
     assert not failed_stories, 'Incorrect story names: %s' % str(failed_stories)
 
 
-def GetDisabledStories(benchmarks):
+def GetDisabledStories(benchmarks, raw_expectations_data):
   # Creates a dictionary of the format:
   # {
   #   'benchmark_name1' : {
@@ -71,7 +72,9 @@ def GetDisabledStories(benchmarks):
   for benchmark in benchmarks:
     name = benchmark.Name()
     disables[name] = {}
-    expectations = benchmark().GetExpectations().AsDict()['stories']
+    b = benchmark()
+    b.AugmentExpectationsWithParser(raw_expectations_data)
+    expectations = b.expectations.AsDict()['stories']
     for story in expectations:
       for conditions, reason in  expectations[story]:
         if not disables[name].get(story):
@@ -89,11 +92,12 @@ def main(args):
       help=('Prints list of disabled stories.'))
   options = parser.parse_args(args)
   benchmarks = benchmark_finders.GetAllBenchmarks()
-
+  with open(path_util.GetExpectationsPath()) as fp:
+    raw_expectations_data = fp.read()
   if options.list:
-    stories = GetDisabledStories(benchmarks)
+    stories = GetDisabledStories(benchmarks, raw_expectations_data)
     print json.dumps(stories, sort_keys=True, indent=4, separators=(',', ': '))
   else:
-    validate_story_names(benchmarks)
+    validate_story_names(benchmarks, raw_expectations_data)
     check_decorators(benchmarks)
   return 0

tools/perf/core/story_expectation_validator_unittest.py

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import unittest
+
+from core import story_expectation_validator
+
+from telemetry import benchmark
+from telemetry import story
+
+class FakePage(object):
+  def __init__(self, name):
+    self._name = name
+
+  @property
+  def name(self):
+    return self._name
+
+
+class FakeStorySetOne(story.StorySet):
+  def __init__(self): # pylint: disable=super-init-not-called
+    self._stories = [
+        FakePage('One'),
+        FakePage('Two')
+    ]
+
+  @property
+  def stories(self):
+    return self._stories
+
+
+class FakeBenchmark(benchmark.Benchmark):
+  @classmethod
+  def Name(cls):
+    return 'b1'
+
+  def CreateStorySet(self, options):
+    return FakeStorySetOne()
+
+
+class StoryExpectationValidatorTest(unittest.TestCase):
+  def testValidateStoryInValidName(self):
+    raw_expectations = '# tags: Mac\ncrbug.com/123 [ Mac ] b1/s1 [ Skip ]'
+    benchmarks = [FakeBenchmark]
+    with self.assertRaises(AssertionError):
+      story_expectation_validator.validate_story_names(
+          benchmarks, raw_expectations)
+
+  def testValidateStoryValidName(self):
+    raw_expectations = '# tags: Mac\ncrbug.com/123 [ Mac ] b1/One [ Skip ]'
+    benchmarks = [FakeBenchmark]
+    # If a name is invalid, an exception is thrown. If no exception is thrown
+    # all story names are valid. That is why there is no assert here.
+    story_expectation_validator.validate_story_names(
+        benchmarks, raw_expectations)
+
+  def testGetDisabledStoriesWithExpectationsData(self):
+    raw_expectations = '# tags: Mac\ncrbug.com/123 [ Mac ] b1/One [ Skip ]'
+    benchmarks = [FakeBenchmark]
+    results = story_expectation_validator.GetDisabledStories(
+        benchmarks, raw_expectations)
+    expected = {'b1': {'One': [(['Mac'], 'crbug.com/123')]}}
+    self.assertEqual(expected, results)
+
+  def testGetDisabledStoriesWithoutMatchingExpectationsData(self):
+    raw_expectations = '# tags: Mac\ncrbug.com/123 [ Mac ] b2/One [ Skip ]'
+    benchmarks = [FakeBenchmark]
+    results = story_expectation_validator.GetDisabledStories(
+        benchmarks, raw_expectations)
+    expected = { 'b1': {}}
+    self.assertEqual(expected, results)