Mojo: Make the mojob.sh unit test runner skip previously-passed tests.

It works for me on my Linux box. It only supports binary tests, not
Python. :/

R=darin@chromium.org

Review URL: https://codereview.chromium.org/217043005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@260321 0039d316-1c4b-4281-b951-d872f2087c98

mojo/tools/data/unittests

+# This file contains a list of Mojo gtest unit tests.
+
+# System tests:
+mojo_system_unittests
+
+# Public tests:
+mojo_public_bindings_unittests
+mojo_public_environment_unittests
+mojo_public_system_unittests
+mojo_public_utility_unittests
+
+# Non-system, non-public tests:
+mojo_apps_js_unittests
+mojo_common_unittests
+mojo_js_unittests
+mojo_service_manager_unittests

mojo/tools/mojob.sh

@@ -50,15 +50,8 @@ do_build() {
 
 do_unittests() {
   echo "Running unit tests in out/$1 ..."
-  "out/$1/mojo_common_unittests" || exit 1
-  "out/$1/mojo_apps_js_unittests" || exit 1
-  "out/$1/mojo_js_unittests" || exit 1
-  "out/$1/mojo_public_bindings_unittests" || exit 1
-  "out/$1/mojo_public_environment_unittests" || exit 1
-  "out/$1/mojo_public_system_unittests" || exit 1
-  "out/$1/mojo_public_utility_unittests" || exit 1
-  "out/$1/mojo_service_manager_unittests" || exit 1
-  "out/$1/mojo_system_unittests" || exit 1
+  mojo/tools/test_runner.py mojo/tools/data/unittests "out/$1" \
+      mojob_test_successes || exit 1
 }
 
 do_perftests() {

mojo/tools/pylib/transitive_hash.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import subprocess
+import sys
+
+from hashlib import sha256
+from os.path import basename, realpath
+
+_logging = logging.getLogger()
+
+# Based on/taken from
+#   http://code.activestate.com/recipes/578231-probably-the-fastest-memoization-decorator-in-the-/
+# (with cosmetic changes).
+def _memoize(f):
+  """Memoization decorator for a function taking a single argument."""
+  class Memoize(dict):
+    def __missing__(self, key):
+      rv = self[key] = f(key)
+      return rv
+  return Memoize().__getitem__
+
+@_memoize
+def _file_hash(filename):
+  """Returns a string representing the hash of the given file."""
+  _logging.debug("Hashing %s ...", filename)
+  rv = subprocess.check_output(['sha256sum', '-b', filename]).split(None, 1)[0]
+  _logging.debug("  => %s", rv)
+  return rv
+
+@_memoize
+def _get_dependencies(filename):
+  """Returns a list of filenames for files that the given file depends on."""
+  _logging.debug("Getting dependencies for %s ...", filename)
+  lines = subprocess.check_output(['ldd', filename]).splitlines()
+  rv = []
+  for line in lines:
+    i = line.find('/')
+    if i < 0:
+      _logging.debug("  => no file found in line: %s", line)
+      continue
+    rv.append(line[i:].split(None, 1)[0])
+  _logging.debug("  => %s", rv)
+  return rv
+
+def transitive_hash(filename):
+  """Returns a string that represents the "transitive" hash of the given
+  file. The transitive hash is a hash of the file and all the shared libraries
+  on which it depends (done in an order-independent way)."""
+  hashes = set()
+  to_hash = [filename]
+  while to_hash:
+    current_filename = realpath(to_hash.pop())
+    current_hash = _file_hash(current_filename)
+    if current_hash in hashes:
+      _logging.debug("Already seen %s (%s) ...", current_filename, current_hash)
+      continue
+    _logging.debug("Haven't seen %s (%s) ...", current_filename, current_hash)
+    hashes.add(current_hash)
+    to_hash.extend(_get_dependencies(current_filename))
+  return sha256('|'.join(sorted(hashes))).hexdigest()
+
+def main(argv):
+  logging.basicConfig()
+  # Uncomment to debug:
+  # _logging.setLevel(logging.DEBUG)
+
+  if len(argv) < 2:
+    print """\
+Usage: %s [file] ...
+
+Prints the \"transitive\" hash of each (executable) file. The transitive
+hash is a hash of the file and all the shared libraries on which it
+depends (done in an order-independent way).""" % basename(argv[0])
+    return 0
+
+  rv = 0
+  for filename in argv[1:]:
+    try:
+      print transitive_hash(filename), filename
+    except:
+      print "ERROR", filename
+      rv = 1
+  return rv
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

mojo/tools/test_runner.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A "smart" test runner for gtest unit tests (that caches successes)."""
+
+import logging
+import os
+import subprocess
+import sys
+
+_logging = logging.getLogger()
+
+_script_dir = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(_script_dir, "pylib"))
+
+from transitive_hash import transitive_hash
+
+def main(argv):
+  logging.basicConfig()
+  # Uncomment to debug:
+  # _logging.setLevel(logging.DEBUG)
+
+  if len(argv) < 3 or len(argv) > 4:
+    print "Usage: %s gtest_list_file root_dir [successes_cache_file]" % \
+        os.path.basename(argv[0])
+    return 0 if len(argv) < 2 else 1
+
+  _logging.debug("Test list file: %s", argv[1])
+  with open(argv[1], 'rb') as f:
+    gtest_list = [y for y in [x.strip() for x in f.readlines()] \
+                      if y and y[0] != '#']
+  _logging.debug("Test list: %s" % gtest_list)
+
+  print "Running tests in directory: %s" % argv[2]
+  os.chdir(argv[2])
+
+  if len(argv) == 4 and argv[3]:
+    successes_cache_filename = argv[3]
+    print "Successes cache file: %s" % successes_cache_filename
+  else:
+    successes_cache_filename = None
+    print "No successes cache file (will run all tests unconditionally)"
+
+  if successes_cache_filename:
+    # This file simply contains a list of transitive hashes of tests that
+    # succeeded.
+    try:
+      _logging.debug("Trying to read successes cache file: %s",
+                     successes_cache_filename)
+      with open(argv[3], 'rb') as f:
+        successes = set([x.strip() for x in f.readlines()])
+      _logging.debug("Successes: %s", successes)
+    except:
+      # Just assume that it didn't exist, or whatever.
+      print "Failed to read successes cache file %s (will create)" % argv[3]
+      successes = set()
+
+  # Run gtests with color if we're on a TTY (and we're not being told explicitly
+  # what to do).
+  if sys.stdout.isatty() and 'GTEST_COLOR' not in os.environ:
+    _logging.debug("Setting GTEST_COLOR=yes")
+    os.environ['GTEST_COLOR'] = 'yes'
+
+  # TODO(vtl): We may not close this file on failure.
+  successes_cache_file = open(successes_cache_filename, 'ab') \
+      if successes_cache_filename else None
+  for gtest in gtest_list:
+    if successes_cache_file:
+      _logging.debug("Getting transitive hash for %s ... " % gtest)
+      try:
+        gtest_hash = transitive_hash(gtest)
+      except:
+        print "Failed to get transitive hash for %s" % gtest
+        return 1
+      _logging.debug("  Transitive hash: %s" % gtest_hash)
+
+      if gtest_hash in successes:
+        print "Skipping %s (previously succeeded)" % gtest
+        continue
+
+    print "Running %s ..." % gtest
+    try:
+      subprocess.check_output(["./" + gtest], stderr=subprocess.STDOUT)
+      print "  Succeeded"
+      # Record success.
+      if successes_cache_filename:
+        successes.add(gtest_hash)
+        successes_cache_file.write(gtest_hash + '\n')
+        successes_cache_file.flush()
+    except subprocess.CalledProcessError as e:
+      print "  Failed with exit code %d and output:" % e.returncode
+      print 72 * '-'
+      print e.output
+      print 72 * '-'
+      return 1
+    except OSError as e:
+      print "  Failed to start test"
+      return 1
+  print "All tests succeeded"
+  if successes_cache_file:
+    successes_cache_file.close()
+
+  return 0
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))