Add src-side launcher for telemetry_unittests

This will need a recipe-side change to pass the paths known to recipe
(from the path recipe module) and properties (from the properties
recipe module).

Each of these key-value pairs is passed as two command line arguments,
so that parsing them is easy and rock solid (avoids escaping issues),
and is also easy to convert to dict python-side.

BUG=422235
R=iannucci@chromium.org

Review URL: https://codereview.chromium.org/649683005

Cr-Commit-Position: refs/heads/master@{#300659}

testing/scripts/checkdeps.py

@@ -3,41 +3,18 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import argparse
-import contextlib
 import json
 import os
-import subprocess
 import sys
-import tempfile
 
 
-SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
-SRC_DIR = os.path.abspath(
-    os.path.join(SCRIPT_DIR, os.path.pardir, os.path.pardir))
+import common
 
 
-def run_command(argv):
-  print 'Running %r' % argv
-  rc = subprocess.call(argv)
-  print 'Command %r returned exit code %d' % (argv, rc)
-  return rc
-
-
-@contextlib.contextmanager
-def temporary_file():
-  fd, path = tempfile.mkstemp()
-  os.close(fd)
-  try:
-    yield path
-  finally:
-    os.remove(path)
-
-
-def mode_run(args):
-  with temporary_file() as tempfile_path:
-    rc = run_command([
-        os.path.join(SRC_DIR, 'buildtools', 'checkdeps', 'checkdeps.py'),
+def main_run(args):
+  with common.temporary_file() as tempfile_path:
+    rc = common.run_command([
+        os.path.join(common.SRC_DIR, 'buildtools', 'checkdeps', 'checkdeps.py'),
         '--json', tempfile_path
     ])
 
@@ -49,27 +26,21 @@ def mode_run(args):
     for violation in result['violations']:
       result_set.add((result['dependee_path'], violation['include_path']))
 
-  with open(args.output, 'w') as f:
-    json.dump({
-        'valid': True,
-        'failures': ['%s: %s' % (r[0], r[1]) for r in result_set],
-    }, f)
+  json.dump({
+      'valid': True,
+      'failures': ['%s: %s' % (r[0], r[1]) for r in result_set],
+  }, args.output)
 
   return rc
 
 
-def main(argv):
-  parser = argparse.ArgumentParser()
-
-  subparsers = parser.add_subparsers()
-
-  run_parser = subparsers.add_parser('run')
-  run_parser.add_argument('--output', required=True)
-  run_parser.set_defaults(func=mode_run)
-
-  args = parser.parse_args(argv)
-  return args.func(args)
+def main_compile_targets(args):
+  json.dump([], args.output)
 
 
 if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
+  funcs = {
+    'run': main_run,
+    'compile_targets': main_compile_targets,
+  }
+  sys.exit(common.run_script(sys.argv[1:], funcs))

testing/scripts/common.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import contextlib
+import json
+import os
+import subprocess
+import tempfile
+
+
+SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
+SRC_DIR = os.path.abspath(
+    os.path.join(SCRIPT_DIR, os.path.pardir, os.path.pardir))
+
+
+# run-webkit-tests returns the number of failures as the return
+# code, but caps the return code at 101 to avoid overflow or colliding
+# with reserved values from the shell.
+MAX_FAILURES_EXIT_STATUS = 101
+
+
+def run_script(argv, funcs):
+  def parse_json(path):
+    with open(path) as f:
+      return json.load(f)
+  parser = argparse.ArgumentParser()
+  # TODO(phajdan.jr): Make build-config-fs required after passing it in recipe.
+  parser.add_argument('--build-config-fs')
+  parser.add_argument('--paths', type=parse_json, default={})
+  parser.add_argument('--properties', type=parse_json, default={})
+
+  subparsers = parser.add_subparsers()
+
+  run_parser = subparsers.add_parser('run')
+  run_parser.add_argument(
+      '--output', type=argparse.FileType('w'), required=True)
+  run_parser.add_argument('--filter-file', type=argparse.FileType('r'))
+  run_parser.set_defaults(func=funcs['run'])
+
+  run_parser = subparsers.add_parser('compile_targets')
+  run_parser.add_argument(
+      '--output', type=argparse.FileType('w'), required=True)
+  run_parser.set_defaults(func=funcs['compile_targets'])
+
+  args = parser.parse_args(argv)
+  return args.func(args)
+
+
+def run_command(argv):
+  print 'Running %r' % argv
+  rc = subprocess.call(argv)
+  print 'Command %r returned exit code %d' % (argv, rc)
+  return rc
+
+
+@contextlib.contextmanager
+def temporary_file():
+  fd, path = tempfile.mkstemp()
+  os.close(fd)
+  try:
+    yield path
+  finally:
+    os.remove(path)
+
+
+def parse_common_test_results(json_results):
+  def convert_trie_to_flat_paths(trie, prefix=None):
+    # Also see webkitpy.layout_tests.layout_package.json_results_generator
+    result = {}
+    for name, data in trie.iteritems():
+      if prefix:
+        name = prefix + '/' + name
+      if len(data) and not 'actual' in data and not 'expected' in data:
+        result.update(convert_trie_to_flat_paths(data, name))
+      else:
+        result[name] = data
+    return result
+
+  results = {
+    'passes': {},
+    'unexpected_passes': {},
+    'failures': {},
+    'unexpected_failures': {},
+    'flakes': {},
+    'unexpected_flakes': {},
+  }
+
+  # TODO(dpranke): crbug.com/357866 - we should simplify the handling of
+  # both the return code and parsing the actual results, below.
+
+  passing_statuses = ('PASS', 'SLOW', 'NEEDSREBASELINE',
+                        'NEEDSMANUALREBASELINE')
+
+  for test, result in convert_trie_to_flat_paths(
+      json_results['tests']).iteritems():
+    key = 'unexpected_' if result.get('is_unexpected') else ''
+    data = result['actual']
+    actual_results = data.split()
+    last_result = actual_results[-1]
+    expected_results = result['expected'].split()
+
+    if (len(actual_results) > 1 and
+        (last_result in expected_results or last_result in passing_statuses)):
+      key += 'flakes'
+    elif last_result in passing_statuses:
+      key += 'passes'
+      # TODO(dpranke): crbug.com/357867 ...  Why are we assigning result
+      # instead of actual_result here. Do we even need these things to be
+      # hashes, or just lists?
+      data = result
+    else:
+      key += 'failures'
+    results[key][test] = data
+
+  return results

testing/scripts/get_compile_targets.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import json
+import os
+import sys
+
+import common
+
+
+def main(argv):
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--output', required=True)
+  parser.add_argument('args', nargs=argparse.REMAINDER)
+
+  args = parser.parse_args(argv)
+
+  passthrough_args = args.args
+  if passthrough_args[0] == '--':
+    passthrough_args = passthrough_args[1:]
+
+  results = {}
+
+  for filename in os.listdir(common.SCRIPT_DIR):
+    if not filename.endswith('.py'):
+      continue
+    if filename in ('common.py', 'get_compile_targets.py'):
+      continue
+
+    with common.temporary_file() as tempfile_path:
+      rc = common.run_command(
+          [os.path.join(common.SCRIPT_DIR, filename)] +
+          passthrough_args +
+          [
+              'compile_targets',
+              '--output', tempfile_path
+          ]
+      )
+      if rc != 0:
+        return rc
+
+      with open(tempfile_path) as f:
+        results[filename] = json.load(f)
+
+  with open(args.output, 'w') as f:
+    json.dump(results, f)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

testing/scripts/telemetry_unittests.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+import sys
+
+
+import common
+
+
+def main_run(args):
+  filter_tests = []
+  if args.filter_file:
+    filter_tests = json.load(args.filter_file)
+
+  with common.temporary_file() as tempfile_path:
+    rc = common.run_command([
+        os.path.join(args.paths['build'], 'scripts', 'tools', 'runit.py'),
+        '--show-path',
+        os.path.join(args.paths['build'], 'scripts', 'slave', 'runtest.py'),
+        '--target', args.build_config_fs,
+        '--xvfb',
+        '--annotate', 'gtest',
+        '--test-type', 'telemetry_unittests',
+        '--builder-name', args.properties['buildername'],
+        '--slave-name', args.properties['slavename'],
+        '--build-number', str(args.properties['buildnumber']),
+        '--run-python-script',
+        os.path.join(common.SRC_DIR, 'tools', 'telemetry', 'run_tests'),
+        '--browser', args.build_config_fs.lower(),
+        '--retry-limit', '3',
+        '--write-full-results-to', tempfile_path,
+    ] + filter_tests)
+
+    with open(tempfile_path) as f:
+      results = json.load(f)
+
+  parsed_results = common.parse_common_test_results(results)
+  failures = parsed_results['unexpected_failures']
+
+  json.dump({
+      'valid': bool(rc <= common.MAX_FAILURES_EXIT_STATUS and
+                   ((rc == 0) or failures)),
+      'failures': failures.keys(),
+  }, args.output)
+
+  return rc
+
+
+def main_compile_targets(args):
+  json.dump(['chrome'], args.output)
+
+
+if __name__ == '__main__':
+  funcs = {
+    'run': main_run,
+    'compile_targets': main_compile_targets,
+  }
+  sys.exit(common.run_script(sys.argv[1:], funcs))