Revert "Remove perf_expectations"

This reverts commit 9d5ef42a.

Reason for revert: See crbug.com/893254; looks like a typo in the script.

Original change's description:
> Remove perf_expectations
> 
> This is a modified reland of [1] with most of the changes to sizes.py reverted.
> Those changes are still used by the perf dashboard so must be kept around for
> the time being.
> 
> [1] https://chromium.googlesource.com/chromium/src.git/+/5f552b39961e3b36e018fdd7d3d452526bde2e4f
> 
> R=​dpranke
> Bug: 572393
> Cq-Include-Trybots: luci.chromium.try:linux_chromium_archive_rel_ng;luci.chromium.try:mac_chromium_archive_rel_ng;luci.chromium.try:win_archive;luci.chromium.try:win_x64_archive;master.tryserver.chromium.android:android_archive_rel_ng
> 
> Change-Id: I459387d50f4a328192110c7b75983523a14f787d
> Reviewed-on: https://chromium-review.googlesource.com/c/1255303
> Commit-Queue: Thomas Anderson <thomasanderson@chromium.org>
> Reviewed-by: Dirk Pranke <dpranke@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#597614}

TBR=thakis@chromium.org,dpranke@chromium.org,thomasanderson@chromium.org

Change-Id: Ife13ece23ed3513225003bd18fbe27b0bbf16afc
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: 572393
Cq-Include-Trybots: luci.chromium.try:linux_chromium_archive_rel_ng;luci.chromium.try:mac_chromium_archive_rel_ng;luci.chromium.try:win_archive;luci.chromium.try:win_x64_archive;master.tryserver.chromium.android:android_archive_rel_ng
Reviewed-on: https://chromium-review.googlesource.com/c/1268904Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Commit-Queue: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#597633}

docs/static_initializers.md

@@ -9,7 +9,7 @@ http://neugierig.org/software/chromium/notes/2011/08/static-initializers.html
 # How Static Initializers are Checked
 
 * For Linux and Mac:
-  * The expected count is stored in [//infra/scripts/legacy/scripts/slave/chromium/sizes.py](https://cs.chromium.org/chromium/src/infra/scripts/legacy/scripts/slave/chromium/sizes.py)
+  * The expected count is stored in [//tools/perf_expectations/perf_expectations.json](https://cs.chromium.org/chromium/src/tools/perf_expectations/perf_expectations.json)
 * For Android:
   * The expected count is stored in the build target [//chrome/android:monochrome_static_initializers](https://cs.chromium.org/chromium/src/chrome/android/BUILD.gn)
 

infra/scripts/legacy/scripts/slave/chromium/sizes.py

@@ -29,17 +29,10 @@ from slave import build_directory
 SRC_DIR = os.path.abspath(
     os.path.join(os.path.dirname(__file__), '..', '..', '..', '..', '..', '..'))
 
-EXPECTED_LINUX_SI_COUNTS = {
-  'chrome': 8,
-  'nacl_helper': 6,
-  'nacl_helper_bootstrap': 0,
-}
-
 
 class ResultsCollector(object):
   def __init__(self):
     self.results = {}
-    self.failures = []
 
   def add_result(self, name, identifier, value, units):
     assert name not in self.results
@@ -52,9 +45,6 @@ class ResultsCollector(object):
     # Legacy printing, previously used for parsing the text logs.
     print 'RESULT %s: %s= %s %s' % (name, identifier, value, units)
 
-  def add_failure(self, failure):
-    self.failures.append(failure)
-
 
 def get_size(filename):
   return os.stat(filename)[stat.ST_SIZE]
@@ -175,10 +165,6 @@ def main_mac(options, args, results_collector):
       # For Release builds only, use dump-static-initializers.py to print the
       # list of static initializers.
       if si_count > 0 and options.target == 'Release':
-        result = 125
-        results_collector.add_failure(
-            'Expected 0 static initializers in %s, but found %d' %
-            (chromium_framework_executable, si_count))
         print '\n# Static initializers in %s:' % chromium_framework_executable
 
         # First look for a dSYM to get information about the initializers. If
@@ -244,7 +230,7 @@ def main_mac(options, args, results_collector):
   return 66
 
 
-def check_linux_binary(target_dir, binary_name, options, results_collector):
+def check_linux_binary(target_dir, binary_name, options):
   """Collect appropriate size information about the built Linux binary given.
 
   Returns a tuple (result, sizes).  result is the first non-zero exit
@@ -316,23 +302,16 @@ def check_linux_binary(target_dir, binary_name, options, results_collector):
 
   # For Release builds only, use dump-static-initializers.py to print the list
   # of static initializers.
-  if options.target == 'Release':
-    if (binary_name in EXPECTED_LINUX_SI_COUNTS and
-        si_count > EXPECTED_LINUX_SI_COUNTS[binary_name]):
-      result = 125
-      results_collector.add_failure(
-          'Expected <= %d static initializers in %s, but found %d' %
-          (EXPECTED_LINUX_SI_COUNTS[binary_name], binary_name, si_count))
-    if si_count > 0:
-      build_dir = os.path.dirname(target_dir)
-      dump_static_initializers = os.path.join(os.path.dirname(build_dir),
-                                              'tools', 'linux',
-                                              'dump-static-initializers.py')
-      result, stdout = run_process(result, [dump_static_initializers,
-                                            '-d', binary_file])
-      print '\n# Static initializers in %s:' % binary_file
-      print_si_fail_hint('tools/linux/dump-static-initializers.py')
-      print stdout
+  if si_count > 0 and options.target == 'Release':
+    build_dir = os.path.dirname(target_dir)
+    dump_static_initializers = os.path.join(os.path.dirname(build_dir),
+                                            'tools', 'linux',
+                                            'dump-static-initializers.py')
+    result, stdout = run_process(result, [dump_static_initializers,
+                                          '-d', binary_file])
+    print '\n# Static initializers in %s:' % binary_file
+    print_si_fail_hint('tools/linux/dump-static-initializers.py')
+    print stdout
 
   # Determine if the binary has the DT_TEXTREL marker.
   result, stdout = run_process(result, ['readelf', '-Wd', binary_file])
@@ -371,8 +350,7 @@ def main_linux(options, args, results_collector):
   totals = {}
 
   for binary in binaries:
-    this_result, this_sizes = check_linux_binary(target_dir, binary, options,
-                                                 results_collector)
+    this_result, this_sizes = check_linux_binary(target_dir, binary, options)
     if result == 0:
       result = this_result
     for name, identifier, totals_id, value, units in this_sizes:
@@ -556,8 +534,6 @@ def main():
                            help='specify platform (%s) [default: %%default]'
                                 % ', '.join(platforms))
   option_parser.add_option('--json', help='Path to JSON output file')
-  option_parser.add_option('--failures',
-                           help='Path to JSON output file for failures')
 
   options, args = option_parser.parse_args()
 
@@ -578,10 +554,6 @@ def main():
     with open(options.json, 'w') as f:
       json.dump(results_collector.results, f)
 
-  if options.failures:
-    with open(options.failures, 'w') as f:
-      json.dump(results_collector.failures, f)
-
   return rc
 
 

testing/scripts/sizes.py

@@ -44,19 +44,51 @@ def main_run(script_args):
         os.path.join(
             common.SRC_DIR, 'infra', 'scripts', 'legacy', 'scripts', 'slave',
             'chromium', 'sizes.py'),
-        '--failures', tempfile_path
+        '--json', tempfile_path
     ]
     if args.platform:
       sizes_cmd.extend(['--platform', args.platform])
     rc = common.run_runtest(script_args, runtest_args + sizes_cmd)
     with open(tempfile_path) as f:
-      failures = json.load(f)
+      results = json.load(f)
+
+  with open(os.path.join(common.SRC_DIR, 'tools', 'perf_expectations',
+                         'perf_expectations.json')) as f:
+    perf_expectations = json.load(f)
+
+  valid = (rc == 0)
+  failures = []
+
+  for name, result in results.iteritems():
+    fqtn = '%s/%s/%s' % (args.prefix, name, result['identifier'])
+    if fqtn not in perf_expectations:
+      continue
+
+    if perf_expectations[fqtn]['type'] != 'absolute':
+      print 'ERROR: perf expectation %r is not yet supported' % fqtn
+      valid = False
+      continue
+
+    actual = result['value']
+    expected = perf_expectations[fqtn]['regress']
+    better = perf_expectations[fqtn]['better']
+    check_result = ((actual <= expected) if better == 'lower'
+                    else (actual >= expected))
+
+    if not check_result:
+      failures.append(fqtn)
+      print 'FAILED %s: actual %s, expected %s, better %s' % (
+          fqtn, actual, expected, better)
 
   json.dump({
-      'valid': (rc == 0 or rc == 125),
+      'valid': valid,
       'failures': failures,
   }, script_args.output)
 
+  # sizes.py itself doesn't fail on regressions.
+  if failures and rc == 0:
+    rc = 1
+
   return rc
 
 

tools/linux/dump-static-initializers.py

@@ -38,10 +38,8 @@ NOTES = {
 IS_GIT_WORKSPACE = (subprocess.Popen(
     ['git', 'rev-parse'], stderr=subprocess.PIPE).wait() == 0)
 
-
 class Demangler(object):
   """A wrapper around c++filt to provide a function to demangle symbols."""
-
   def __init__(self, toolchain):
     self.cppfilt = subprocess.Popen([toolchain + 'c++filt'],
                                     stdin=subprocess.PIPE,
@@ -52,7 +50,6 @@ class Demangler(object):
     self.cppfilt.stdin.write(sym + '\n')
     return self.cppfilt.stdout.readline().strip()
 
-
 # Matches for example: "cert_logger.pb.cc", capturing "cert_logger".
 protobuf_filename_re = re.compile(r'(.*)\.pb\.cc$')
 def QualifyFilenameAsProto(filename):
@@ -75,7 +72,6 @@ def QualifyFilenameAsProto(filename):
     candidate = line.strip()
   return candidate
 
-
 # Regex matching the substring of a symbol's demangled text representation most
 # likely to appear in a source file.
 # Example: "v8::internal::Builtins::InitBuiltinFunctionTable()" becomes
@@ -103,7 +99,6 @@ def QualifyFilename(filename, symbol):
     candidate = line.strip()
   return candidate
 
-
 # Regex matching nm output for the symbols we're interested in.
 # See test_ParseNmLine for examples.
 nm_re = re.compile(r'(\S+) (\S+) t (?:_ZN12)?_GLOBAL__(?:sub_)?I_(.*)')
@@ -128,7 +123,6 @@ def test_ParseNmLine():
     '_GLOBAL__sub_I_extension_specifics.pb.cc')
   assert parse == ('extension_specifics.pb.cc', 40607408, 36), parse
 
-
 # Just always run the test; it is fast enough.
 test_ParseNmLine()
 
@@ -142,7 +136,6 @@ def ParseNm(toolchain, binary):
     if parse:
       yield parse
 
-
 # Regex matching objdump output for the symbols we're interested in.
 # Example line:
 #     12354ab:  (disassembly, including <FunctionReference>)
@@ -165,14 +158,13 @@ def ExtractSymbolReferences(toolchain, binary, start, end):
       if ref.startswith('.LC') or ref.startswith('_DYNAMIC'):
         # Ignore these, they are uninformative.
         continue
-      if re.match('_GLOBAL__(?:sub_)?I_', ref):
+      if ref.startswith('_GLOBAL__I_'):
         # Probably a relative jump within this function.
         continue
       refs.add(ref)
 
   return sorted(refs)
 
-
 def main():
   parser = optparse.OptionParser(usage='%prog [option] filename')
   parser.add_option('-d', '--diffable', dest='diffable',
@@ -244,6 +236,5 @@ def main():
 
   return 0
 
-
 if '__main__' == __name__:
   sys.exit(main())

tools/perf_expectations/OWNERS

+jochen@chromium.org
+thakis@chromium.org
+thestig@chromium.org

tools/perf_expectations/PRESUBMIT.py

+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Presubmit script for perf_expectations.
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts for
+details on the presubmit API built into depot_tools.
+"""
+
+PERF_EXPECTATIONS = 'tools/perf_expectations/perf_expectations.json'
+CONFIG_FILE = 'tools/perf_expectations/chromium_perf_expectations.cfg'
+
+def CheckChangeOnUpload(input_api, output_api):
+  run_tests = False
+  for path in input_api.LocalPaths():
+    path = path.replace('\\', '/')
+    if (PERF_EXPECTATIONS == path or CONFIG_FILE == path):
+      run_tests = True
+
+  output = []
+  if run_tests:
+    whitelist = [r'.+_unittest\.py$']
+    output.extend(input_api.canned_checks.RunUnitTestsInDirectory(
+        input_api, output_api, 'tests', whitelist))
+  return output
+
+
+def CheckChangeOnCommit(input_api, output_api):
+  run_tests = False
+  for path in input_api.LocalPaths():
+    path = path.replace('\\', '/')
+    if (PERF_EXPECTATIONS == path or CONFIG_FILE == path):
+      run_tests = True
+
+  output = []
+  if run_tests:
+    whitelist = [r'.+_unittest\.py$']
+    output.extend(input_api.canned_checks.RunUnitTestsInDirectory(
+        input_api, output_api, 'tests', whitelist))
+
+  output.extend(input_api.canned_checks.CheckDoNotSubmit(input_api,
+                                                         output_api))
+  return output

tools/perf_expectations/README.txt

+This is where old perf machinery used to live, keeping track of binary sizes,
+etc. Now that lives elsewhere and has a team to support it (see
+https://www.chromium.org/developers/tree-sheriffs/perf-sheriffs). This code
+remains to ensure that no static initializers get into Chromium.
+
+Because this code has this history, it's far more complicated than it needs to
+be. TODO(dpranke): Simplify it. https://crbug.com/572393
+
+In the meanwhile, if you're trying to update perf_expectations.json, there are
+no instructions for doing so, and the tools that you used to use don't work
+because they rely on data files that were last updated at the end of 2015. So
+here's what to do to reset the expected static initializer count value.
+
+The expected static initializer count value is in the "regress" field for the
+platform. In addition, each platform has a checksum in the "sha1" field to
+ensure that you properly used the magic tools. Since the magic tools don't work
+anymore, dpranke added a bypass to the verification. If you run:
+
+> tools/perf_expectations/make_expectations.py --checksum --verbose
+
+the script will tell you what the checksum *should* be. Alter the "sha1" field
+to be that value, and you can commit changes to that file.
+
+Please see https://crbug.com/572393 for more information.

tools/perf_expectations/chromium_perf_expectations.cfg

+{
+  "base_url": "http://build.chromium.org/f/chromium/perf",
+  "perf_file": "perf_expectations.json"
+}

tools/perf_expectations/perf_expectations.json

+{"linux-release-64/sizes/chrome-si/initializers": {"reva": 480969, "revb": 480969, "type": "absolute", "better": "lower", "improve": 8, "regress": 8, "tolerance": 0, "sha1": "3c815259"},
+ "linux-release-64/sizes/nacl_helper-si/initializers": {"reva": 480969, "revb": 480969, "type": "absolute", "better": "lower", "improve": 6, "regress": 8, "sha1": "8416f450"},
+ "linux-release-64/sizes/nacl_helper_bootstrap-si/initializers": {"reva": 114822, "revb": 115019, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "228221af"},
+ "linux-release/sizes/chrome-si/initializers": {"reva": 480969, "revb": 480969, "type": "absolute", "better": "lower", "improve": 9, "regress": 9, "tolerance": 0, "sha1": "03dc3cfd"},
+ "linux-release/sizes/nacl_helper-si/initializers": {"reva": 480969, "revb": 480969, "type": "absolute", "better": "lower", "improve": 7, "regress": 9, "sha1": "1a3c5b2b"},
+ "linux-release/sizes/nacl_helper_bootstrap-si/initializers": {"reva": 114822, "revb": 115019, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "dd908f29"},
+ "mac-release/sizes/chrome-si/initializers": {"reva": 281731, "revb": 281731, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "tolerance": 0, "sha1": "01759b7f"},
+ "load": true
+}

tools/perf_expectations/sample_test_cases.json

+{"linux-release/media_tests_av_perf/audio_latency/latency": {"reva": 180005, "revb": 180520, "type": "absolute", "better": "lower", "improve": 190, "regress": 222, "sha1": "fc9815d5"},
+"linux-release/media_tests_av_perf/dropped_fps/tulip2.wav": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "fb8157f9"},
+"linux-release/media_tests_av_perf/dropped_fps/tulip2.webm": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "c0fb3421"},
+"linux-release/media_tests_av_perf/dropped_frames/crowd1080.webm": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "fa9582d3"},
+"linux-release/media_tests_av_perf/dropped_frames/crowd2160.webm": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 166, "regress": 231, "sha1": "ca3a7a47"},
+"linux-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 34239, "revb": 1298213, "type": "absolute", "better": "higher", "improve": 0, "regress": 0},
+"linux-release/media_tests_av_perf/fps/tulip2.mp3": {"reva": 34239, "revb": 1298213, "type": "absolute", "better": "higher", "improve": 0, "regress": 0},
+"linux-release/media_tests_av_perf/fps/tulip2.mp4": {"reva": 34239, "revb": 1298213, "type": "absolute", "better": "higher", "improve": 32, "regress": 28},
+"linux-release/media_tests_av_perf/fps/tulip2.ogg": {"reva": 34239, "revb": 1298213, "type": "absolute", "better": "higher", "improve": 0, "regress": 0},
+"linux-release/media_tests_av_perf/fps/tulip2.ogv": {"reva": 34239, "revb": 1298213, "type": "absolute", "better": "higher", "improve": 32, "regress": 28},
+"linux-release/media_tests_av_perf/fps/tulip2.wav": {"reva": 34239, "revb": 1298213, "type": "absolute", "better": "higher", "improve": 0, "regress": 0},
+"win-release/media_tests_av_perf/dropped_fps/tulip2.wav": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "646c02f2"},
+"win-release/media_tests_av_perf/dropped_fps/tulip2.webm": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "46c97b57"},
+"win-release/media_tests_av_perf/dropped_frames/crowd1080.webm": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 0, "regress": 0, "sha1": "9b709aab"},
+"win-release/media_tests_av_perf/dropped_frames/crowd2160.webm": {"reva": 181131, "revb": 181572, "type": "absolute", "better": "lower", "improve": 174, "regress": 204, "sha1": "4c0270a6"},
+"win-release/media_tests_av_perf/fps/crowd1080.webm": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 53, "regress": 43, "sha1": "7ad49461"},
+"win-release/media_tests_av_perf/fps/crowd2160.webm": {"reva": 176330, "revb": 176978, "type": "absolute", "better": "higher", "improve": 26.0399945997, "regress": 25.9062437562, "sha1": "700526a9"},
+"win-release/media_tests_av_perf/fps/crowd360.webm": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 51, "regress": 47, "sha1": "7f8ef21c"},
+"win-release/media_tests_av_perf/fps/crowd480.webm": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 50, "regress": 47, "sha1": "5dc96881"},
+"win-release/media_tests_av_perf/fps/crowd720.webm": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 52, "regress": 47, "sha1": "4fcfb653"},
+"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 0, "regress": 0, "sha1": "54d94538"},
+"win-release/media_tests_av_perf/fps/tulip2.mp3": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 0, "regress": 0, "sha1": "113aef17"},
+"win-release/media_tests_av_perf/fps/tulip2.mp4": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 30, "regress": 28, "sha1": "a22847d0"},
+"win-release/media_tests_av_perf/fps/tulip2.ogg": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 0, "regress": 0, "sha1": "6ee2e716"},
+"win-release/media_tests_av_perf/fps/tulip2.ogv": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 32, "regress": 26, "sha1": "dfadb872"},
+"win-release/media_tests_av_perf/fps/tulip2.wav": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 0, "regress": 0, "sha1": "530c5bf5"},
+"win-release/media_tests_av_perf/fps/tulip2.webm": {"reva": 163299, "revb": 164141, "type": "absolute", "better": "higher", "improve": 30, "regress": 28, "sha1": "35b91c8e"}
+}
\ No newline at end of file

tools/perf_expectations/tests/perf_expectations_unittest.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Verify perf_expectations.json can be loaded using simplejson.
+
+perf_expectations.json is a JSON-formatted file.  This script verifies
+that simplejson can load it correctly.  It should catch most common
+formatting problems.
+"""
+
+import subprocess
+import sys
+import os
+import unittest
+import re
+
+simplejson = None
+
+def OnTestsLoad():
+  old_path = sys.path
+  script_path = os.path.dirname(sys.argv[0])
+  load_path = None
+  global simplejson
+
+  # This test script should be stored in src/tools/perf_expectations/.  That
+  # directory will most commonly live in 2 locations:
+  #
+  #   - a regular Chromium checkout, in which case src/third_party
+  #     is where to look for simplejson
+  #
+  #   - a buildbot checkout, in which case .../pylibs is where
+  #     to look for simplejson
+  #
+  # Locate and install the correct path based on what we can find.
+  #
+  for path in ('../../../third_party', '../../../../../pylibs'):
+    path = os.path.join(script_path, path)
+    if os.path.exists(path) and os.path.isdir(path):
+      load_path = os.path.abspath(path)
+      break
+
+  if load_path is None:
+    msg = "%s expects to live within a Chromium checkout" % sys.argv[0]
+    raise Exception, "Error locating simplejson load path (%s)" % msg
+
+  # Try importing simplejson once.  If this succeeds, we found it and will
+  # load it again later properly.  Fail if we cannot load it.
+  sys.path.append(load_path)
+  try:
+    import simplejson as Simplejson
+    simplejson = Simplejson
+  except ImportError, e:
+    msg = "%s expects to live within a Chromium checkout" % sys.argv[0]
+    raise Exception, "Error trying to import simplejson from %s (%s)" % \
+                     (load_path, msg)
+  finally:
+    sys.path = old_path
+  return True
+
+def LoadJsonFile(filename):
+  f = open(filename, 'r')
+  try:
+    data = simplejson.load(f)
+  except ValueError, e:
+    f.seek(0)
+    print "Error reading %s:\n%s" % (filename,
+                                     f.read()[:50]+'...')
+    raise e
+  f.close()
+  return data
+
+OnTestsLoad()
+
+CONFIG_JSON = os.path.join(os.path.dirname(sys.argv[0]),
+                           '../chromium_perf_expectations.cfg')
+MAKE_EXPECTATIONS = os.path.join(os.path.dirname(sys.argv[0]),
+                                 '../make_expectations.py')
+PERF_EXPECTATIONS = os.path.join(os.path.dirname(sys.argv[0]),
+                                 '../perf_expectations.json')
+
+
+class PerfExpectationsUnittest(unittest.TestCase):
+  def testPerfExpectations(self):
+    # Test data is dictionary.
+    perf_data = LoadJsonFile(PERF_EXPECTATIONS)
+    if not isinstance(perf_data, dict):
+      raise Exception('perf expectations is not a dict')
+
+    # Test the 'load' key.
+    if not 'load' in perf_data:
+      raise Exception("perf expectations is missing a load key")
+    if not isinstance(perf_data['load'], bool):
+      raise Exception("perf expectations load key has non-bool value")
+
+    # Test all key values are dictionaries.
+    bad_keys = []
+    for key in perf_data:
+      if key == 'load':
+        continue
+      if not isinstance(perf_data[key], dict):
+        bad_keys.append(key)
+    if len(bad_keys) > 0:
+      msg = "perf expectations keys have non-dict values"
+      raise Exception("%s: %s" % (msg, bad_keys))
+
+    # Test all key values have delta and var keys.
+    for key in perf_data:
+      if key == 'load':
+        continue
+
+      # First check if regress/improve is in the key's data.
+      if 'regress' in perf_data[key]:
+        if 'improve' not in perf_data[key]:
+          bad_keys.append(key)
+        if (not isinstance(perf_data[key]['regress'], int) and
+            not isinstance(perf_data[key]['regress'], float)):
+          bad_keys.append(key)
+        if (not isinstance(perf_data[key]['improve'], int) and
+            not isinstance(perf_data[key]['improve'], float)):
+          bad_keys.append(key)
+      else:
+        # Otherwise check if delta/var is in the key's data.
+        if 'delta' not in perf_data[key] or 'var' not in perf_data[key]:
+          bad_keys.append(key)
+        if (not isinstance(perf_data[key]['delta'], int) and
+            not isinstance(perf_data[key]['delta'], float)):
+          bad_keys.append(key)
+        if (not isinstance(perf_data[key]['var'], int) and
+            not isinstance(perf_data[key]['var'], float)):
+          bad_keys.append(key)
+
+    if len(bad_keys) > 0:
+      msg = "perf expectations key values missing or invalid delta/var"
+      raise Exception("%s: %s" % (msg, bad_keys))
+
+    # Test all keys have the correct format.
+    for key in perf_data:
+      if key == 'load':
+        continue
+      # tools/buildbot/scripts/master/log_parser.py should have a matching
+      # regular expression.
+      if not re.match(r"^([\w\.-]+)/([\w\.-]+)/([\w\.-]+)/([\w\.-]+)$", key):
+        bad_keys.append(key)
+    if len(bad_keys) > 0:
+      msg = "perf expectations keys in bad format, expected a/b/c/d"
+      raise Exception("%s: %s" % (msg, bad_keys))
+
+  def testNoUpdatesNeeded(self):
+    p = subprocess.Popen([MAKE_EXPECTATIONS, '-s'], stdout=subprocess.PIPE)
+    p.wait();
+    self.assertEqual(p.returncode, 0,
+        msg='Update expectations first by running ./make_expectations.py')
+
+  def testConfigFile(self):
+    # Test that the config file can be parsed as JSON.
+    config = LoadJsonFile(CONFIG_JSON)
+    # Require the following keys.
+    if 'base_url' not in config:
+      raise Exception('base_url not specified in config file')
+    if 'perf_file' not in config:
+      raise Exception('perf_file not specified in config file')
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/perf_expectations/update_perf_expectations.py

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Prepare tests that require re-baselining for input to make_expectations.py.
+
+The regularly running perf-AV tests require re-baselineing of expectations
+about once a week. The steps involved in rebaselining are:
+
+1.) Identify the tests to update, based off reported e-mail results.
+2.) Figure out reva and revb values, which is the starting and ending revision
+ numbers for the range that we should use to obtain new thresholds.
+3.) Modify lines in perf_expectations.json referring to the tests to be updated,
+ so that they may be used as input to make_expectations.py.
+
+This script automates the last step above.
+
+Here's a sample line from perf_expectations.json:
+
+"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": 163299, \
+"revb": 164141, "type": "absolute", "better": "higher", "improve": 0, \
+"regress": 0, "sha1": "54d94538"},
+
+To get the above test ready for input to make_expectations.py, it should become:
+
+"win-release/media_tests_av_perf/fps/tulip2.m4a": {"reva": <new reva>, \
+"revb": <new revb>, "type": "absolute", "better": "higher", "improve": 0, \
+"regress": 0},
+
+Examples:
+
+1.) To update the test specified above and get baseline
+values using the revision range 12345 and 23456, run this script with a command
+line like this:
+  python update_perf_expectations.py -f \
+  win-release/media_tests_av_perf/fps/tulip2.m4a --reva 12345 --revb 23456
+Or, using an input file,
+where the input file contains a single line with text
+  win-release/media_tests_av_perf/fps/tulip2.m4a
+run with this command line:
+  python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456
+
+2.) Let's say you want to update all seek tests on windows, and get baseline
+values using the revision range 12345 and 23456.
+Run this script with this command line:
+  python update_perf_expectations.py -f win-release/media_tests_av_perf/seek/ \
+   --reva 12345 --revb 23456
+Or:
+  python update_perf_expectations.py -f win-release/.*/seek/ --reva 12345 \
+  --revb 23456
+
+Or, using an input file,
+where the input file contains a single line with text win-release/.*/seek/:
+  python update_perf_expectations.py -i input.txt --reva 12345 --revb 23456
+
+3.) Similarly, if you want to update seek tests on all platforms
+  python update_perf_expectations.py -f .*-release/.*/seek/ --reva 12345 \
+  --revb 23456
+
+"""
+
+import logging
+from optparse import OptionParser
+import os
+import re
+
+import make_expectations as perf_ex_lib
+
+# Default logging is INFO. Use --verbose to enable DEBUG logging.
+_DEFAULT_LOG_LEVEL = logging.INFO
+
+
+def GetTestsToUpdate(contents, all_test_keys):
+  """Parses input contents and obtains tests to be re-baselined.
+
+  Args:
+    contents: string containing contents of input file.
+    all_test_keys: list of keys of test dictionary.
+  Returns:
+    A list of keys for tests that should be updated.
+  """
+  # Each line of the input file specifies a test case to update.
+  tests_list = []
+  for test_case_filter in contents.splitlines():
+    # Skip any empty lines.
+    if test_case_filter:
+      # Sample expected line:
+      # win-release/media_tests_av_perf/seek/\
+      # CACHED_BUFFERED_SEEK_NoConstraints_crowd1080.ogv
+      # Or, if reg-ex, then sample line:
+      # win-release/media-tests_av_perf/seek*
+      # Skip any leading spaces if they exist in the input file.
+      logging.debug('Trying to match %s', test_case_filter)
+      tests_list.extend(GetMatchingTests(test_case_filter.strip(),
+                                         all_test_keys))
+  return tests_list
+
+
+def GetMatchingTests(tests_to_update, all_test_keys):
+  """Parses input reg-ex filter and obtains tests to be re-baselined.
+
+  Args:
+    tests_to_update: reg-ex string specifying tests to be updated.
+    all_test_keys: list of keys of tests dictionary.
+  Returns:
+    A list of keys for tests that should be updated.
+  """
+  tests_list = []
+  search_string = re.compile(tests_to_update)
+  # Get matching tests from the dictionary of tests
+  for test_key in all_test_keys:
+    if search_string.match(test_key):
+      tests_list.append(test_key)
+      logging.debug('%s will be updated', test_key)
+  logging.info('%s tests found matching reg-ex: %s', len(tests_list),
+               tests_to_update)
+  return tests_list
+
+
+def PrepareTestsForUpdate(tests_to_update, all_tests, reva, revb):
+  """Modifies value of tests that are to re-baselined:
+     Set reva and revb values to specified new values. Remove sha1.
+
+  Args:
+    tests_to_update: list of tests to be updated.
+    all_tests: dictionary of all tests.
+    reva: oldest revision in range to use for new values.
+    revb: newest revision in range to use for new values.
+  Raises:
+    ValueError: If reva or revb are not valid ints, or if either
+    of them are negative.
+  """
+  reva = int(reva)
+  revb = int(revb)
+
+  if reva < 0 or revb < 0:
+    raise ValueError('Revision values should be positive.')
+  # Ensure reva is less than revb.
+  # (this is similar to the check done in make_expectations.py)
+  if revb < reva:
+    temp = revb
+    revb = reva
+    reva = temp
+  for test_key in tests_to_update:
+    # Get original test from the dictionary of tests
+    test_value = all_tests[test_key]
+    if test_value:
+      # Sample line in perf_expectations.json:
+      #  "linux-release/media_tests _av_perf/dropped_frames/crowd360.webm":\
+      # {"reva": 155180, "revb": 155280, "type": "absolute", \
+      # "better": "lower", "improve": 0, "regress": 3, "sha1": "276ba29c"},
+      # Set new revision range
+      test_value['reva'] = reva
+      test_value['revb'] = revb
+      # Remove sha1 to indicate this test requires an update
+      # Check first to make sure it exist.
+      if 'sha1' in test_value:
+        del test_value['sha1']
+    else:
+      logging.warning('%s does not exist.', test_key)
+  logging.info('Done preparing tests for update.')
+
+
+def GetCommandLineOptions():
+  """Parse command line arguments.
+
+  Returns:
+    An options object containing command line arguments and their values.
+  """
+  parser = OptionParser()
+
+  parser.add_option('--reva', dest='reva', type='int',
+                    help='Starting revision of new range.',
+                    metavar='START_REVISION')
+  parser.add_option('--revb', dest='revb', type='int',
+                    help='Ending revision of new range.',
+                    metavar='END_REVISION')
+  parser.add_option('-f', dest='tests_filter',
+                    help='Regex to use for filtering tests to be updated. '
+                    'At least one of -filter or -input_file must be provided. '
+                    'If both are provided, then input-file is used.',
+                    metavar='FILTER', default='')
+  parser.add_option('-i', dest='input_file',
+                    help='Optional path to file with reg-exes for tests to'
+                    ' update. If provided, it overrides the filter argument.',
+                    metavar='INPUT_FILE', default='')
+  parser.add_option('--config', dest='config_file',
+                    default=perf_ex_lib.DEFAULT_CONFIG_FILE,
+                    help='Set the config file to FILE.', metavar='FILE')
+  parser.add_option('-v', dest='verbose', action='store_true', default=False,
+                    help='Enable verbose output.')
+  options = parser.parse_args()[0]
+  return options
+
+
+def Main():
+  """Main driver function."""
+  options = GetCommandLineOptions()
+
+  _SetLogger(options.verbose)
+  # Do some command-line validation
+  if not options.input_file and not options.tests_filter:
+    logging.error('At least one of input-file or test-filter must be provided.')
+    exit(1)
+  if options.input_file and options.tests_filter:
+    logging.error('Specify only one of input file or test-filter.')
+    exit(1)
+  if not options.reva or not options.revb:
+    logging.error('Start and end revision of range must be specified.')
+    exit(1)
+
+  # Load config.
+  config = perf_ex_lib.ConvertJsonIntoDict(
+      perf_ex_lib.ReadFile(options.config_file))
+
+  # Obtain the perf expectations file from the config file.
+  perf_file = os.path.join(
+      os.path.dirname(options.config_file), config['perf_file'])
+
+  # We should have all the information we require now.
+  # On to the real thang.
+  # First, get all the existing tests from the original perf_expectations file.
+  all_tests = perf_ex_lib.ConvertJsonIntoDict(
+      perf_ex_lib.ReadFile(perf_file))
+  all_test_keys = all_tests.keys()
+  # Remove the load key, because we don't want to modify it.
+  all_test_keys.remove('load')
+  # Keep tests sorted, like in the original file.
+  all_test_keys.sort()
+
+  # Next, get all tests that have been identified for an update.
+  tests_to_update = []
+  if options.input_file:
+    # Tests to update have been specified in an input_file.
+    # Get contents of file.
+    tests_filter = perf_ex_lib.ReadFile(options.input_file)
+  elif options.tests_filter:
+    # Tests to update have been specified as a reg-ex filter.
+    tests_filter = options.tests_filter
+
+  # Get tests to update based on filter specified.
+  tests_to_update = GetTestsToUpdate(tests_filter, all_test_keys)
+  logging.info('Done obtaining matching tests.')
+
+  # Now, prepare tests for update.
+  PrepareTestsForUpdate(tests_to_update, all_tests, options.reva, options.revb)
+
+  # Finally, write modified tests back to perf_expectations file.
+  perf_ex_lib.WriteJson(perf_file, all_tests, all_test_keys,
+                        calculate_sha1=False)
+  logging.info('Done writing tests for update to %s.', perf_file)
+
+
+def _SetLogger(verbose):
+  log_level = _DEFAULT_LOG_LEVEL
+  if verbose:
+    log_level = logging.DEBUG
+  logging.basicConfig(level=log_level, format='%(message)s')
+
+
+if __name__ == '__main__':
+  Main()