Upstream: Test scripts for Android (phase 2)

Currently only support run base_unittests

BUG=
TEST=

Review URL: http://codereview.chromium.org/8364020

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@106953 0039d316-1c4b-4281-b951-d872f2087c98

build/android/android_commands.py

@@ -773,5 +773,4 @@ def main(argv):
 
 
 if __name__ == '__main__':
-  print os.path.abspath(os.path.dirname(__file__))
   main(sys.argv)

build/android/base_test_runner.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import os
+
+import android_commands
+from chrome_test_server_spawner import SpawningServer
+from flag_changer import FlagChanger
+import lighttpd_server
+import run_tests_helper
+
+FORWARDER_PATH = '/data/local/tmp/forwarder'
+# These ports must match up with the constants in net/test/test_server.cc
+TEST_SERVER_SPAWNER_PORT = 8001
+TEST_SERVER_PORT = 8002
+TEST_SYNC_SERVER_PORT = 8003
+
+
+class BaseTestRunner(object):
+  """Base class for running tests on a single device."""
+
+  def __init__(self, device):
+    """
+      Args:
+        device: Tests will run on the device of this ID.
+    """
+    self.device = device
+    self.adb = android_commands.AndroidCommands(device=device)
+    # Synchronize date/time between host and device. Otherwise same file on
+    # host and device may have different timestamp which may cause
+    # AndroidCommands.PushIfNeeded failed, or a test which may compare timestamp
+    # got from http head and local time could be failed.
+    self.adb.SynchronizeDateTime()
+    self._http_server = None
+    self._forwarder = None
+    self._spawning_server = None
+    self._spawner_forwarder = None
+    self._forwarder_device_port = 8000
+    self._forwarder_base_url = ('http://localhost:%d' %
+        self._forwarder_device_port)
+    self._flags = FlagChanger(self.adb)
+
+  def RunTests(self):
+    # TODO(bulach): this should actually do SetUp / RunTestsInternal / TearDown.
+    # Refactor the various subclasses to expose a RunTestsInternal without
+    # any params.
+    raise NotImplementedError
+
+  def SetUp(self):
+    """Called before tests run."""
+    pass
+
+  def TearDown(self):
+    """Called when tests finish running."""
+    self.ShutdownHelperToolsForTestSuite()
+
+  def CopyTestData(self, test_data_paths, dest_dir):
+    """Copies |test_data_paths| list of files/directories to |dest_dir|.
+
+    Args:
+      test_data_paths: A list of files or directories relative to |dest_dir|
+          which should be copied to the device. The paths must exist in
+          |CHROME_DIR|.
+      dest_dir: Absolute path to copy to on the device.
+    """
+    for p in test_data_paths:
+      self.adb.PushIfNeeded(
+          os.path.join(run_tests_helper.CHROME_DIR, p),
+          os.path.join(dest_dir, p))
+
+  def LaunchTestHttpServer(self, document_root, extra_config_contents=None):
+    """Launches an HTTP server to serve HTTP tests.
+
+    Args:
+      document_root: Document root of the HTTP server.
+      extra_config_contents: Extra config contents for the HTTP server.
+    """
+    self._http_server = lighttpd_server.LighttpdServer(
+        document_root, extra_config_contents=extra_config_contents)
+    if self._http_server.StartupHttpServer():
+      logging.info('http server started: http://localhost:%s',
+                   self._http_server.port)
+    else:
+      logging.critical('Failed to start http server')
+    # Root access needed to make the forwarder executable work.
+    self.adb.EnableAdbRoot()
+    self.StartForwarderForHttpServer()
+
+  def StartForwarderForHttpServer(self):
+    """Starts a forwarder for the HTTP server.
+
+    The forwarder forwards HTTP requests and responses between host and device.
+    """
+    # Sometimes the forwarder device port may be already used. We have to kill
+    # all forwarder processes to ensure that the forwarder can be started since
+    # currently we can not associate the specified port to related pid.
+    # TODO(yfriedman/wangxianzhu): This doesn't work as most of the time the
+    # port is in use but the forwarder is already dead. Killing all forwarders
+    # is overly destructive and breaks other tests which make use of forwarders.
+    # if IsDevicePortUsed(self.adb, self._forwarder_device_port):
+    #   self.adb.KillAll('forwarder')
+    self._forwarder = run_tests_helper.ForwardDevicePorts(
+        self.adb, [(self._forwarder_device_port, self._http_server.port)])
+
+  def RestartHttpServerForwarderIfNecessary(self):
+    """Restarts the forwarder if it's not open."""
+    # Checks to see if the http server port is being used.  If not forwards the
+    # request.
+    # TODO(dtrainor): This is not always reliable because sometimes the port
+    # will be left open even after the forwarder has been killed.
+    if not run_tests_helper.IsDevicePortUsed(self.adb,
+        self._forwarder_device_port):
+      self.StartForwarderForHttpServer()
+
+  def ShutdownHelperToolsForTestSuite(self):
+    """Shuts down the server and the forwarder."""
+    # Forwarders should be killed before the actual servers they're forwarding
+    # to as they are clients potentially with open connections and to allow for
+    # proper hand-shake/shutdown.
+    if self._forwarder or self._spawner_forwarder:
+      # Kill all forwarders on the device and then kill the process on the host
+      # (if it exists)
+      self.adb.KillAll('forwarder')
+      if self._forwarder:
+        self._forwarder.kill()
+      if self._spawner_forwarder:
+        self._spawner_forwarder.kill()
+    if self._http_server:
+      self._http_server.ShutdownHttpServer()
+    if self._spawning_server:
+      self._spawning_server.Stop()
+    self._flags.Restore()
+
+  def LaunchChromeTestServerSpawner(self):
+    """Launches test server spawner."""
+    self._spawning_server = SpawningServer(TEST_SERVER_SPAWNER_PORT,
+                                          TEST_SERVER_PORT)
+    self._spawning_server.Start()
+    # TODO(yfriedman): Ideally we'll only try to start up a port forwarder if
+    # there isn't one already running but for now we just get an error message
+    # and the existing forwarder still works.
+    self._spawner_forwarder = run_tests_helper.ForwardDevicePorts(
+        self.adb, [(TEST_SERVER_SPAWNER_PORT, TEST_SERVER_SPAWNER_PORT),
+                   (TEST_SERVER_PORT, TEST_SERVER_PORT)])

build/android/chrome_test_server_spawner.py

+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""A "Test Server Spawner" that handles killing/stopping per-test test servers.
+
+It's used to accept requests from the device to spawn and kill instances of the
+chrome test server on the host.
+"""
+
+import BaseHTTPServer
+import logging
+import os
+import sys
+import threading
+import time
+import urlparse
+
+# Path that are needed to import testserver
+cr_src = os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', '..')
+sys.path.append(os.path.join(cr_src, 'third_party'))
+sys.path.append(os.path.join(cr_src, 'third_party', 'tlslite'))
+sys.path.append(os.path.join(cr_src, 'third_party', 'pyftpdlib', 'src'))
+sys.path.append(os.path.join(os.path.abspath(os.path.dirname(__file__)), '..',
+   '..', 'net', 'tools', 'testserver'))
+import testserver
+
+_test_servers = []
+
+class SpawningServerRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+  """A handler used to process http GET request.
+  """
+
+  def GetServerType(self, server_type):
+    """Returns the server type to use when starting the test server.
+
+    This function translate the command-line argument into the appropriate
+    numerical constant.
+    # TODO(yfriedman): Do that translation!
+    """
+    if server_type:
+      pass
+    return 0
+
+  def do_GET(self):
+    parsed_path = urlparse.urlparse(self.path)
+    action = parsed_path.path
+    params = urlparse.parse_qs(parsed_path.query, keep_blank_values=1)
+    logging.info('Action is: %s' % action)
+    if action == '/killserver':
+      # There should only ever be one test server at a time. This may do the
+      # wrong thing if we try and start multiple test servers.
+      _test_servers.pop().Stop()
+    elif action == '/start':
+      logging.info('Handling request to spawn a test webserver')
+      for param in params:
+        logging.info('%s=%s' % (param, params[param][0]))
+      s_type = 0
+      doc_root = None
+      if 'server_type' in params:
+        s_type = self.GetServerType(params['server_type'][0])
+      if 'doc_root' in params:
+        doc_root = params['doc_root'][0]
+      self.webserver_thread = threading.Thread(
+          target=self.SpawnTestWebServer, args=(s_type, doc_root))
+      self.webserver_thread.setDaemon(True)
+      self.webserver_thread.start()
+    self.send_response(200, 'OK')
+    self.send_header('Content-type', 'text/html')
+    self.end_headers()
+    self.wfile.write('<html><head><title>started</title></head></html>')
+    logging.info('Returned OK!!!')
+
+  def SpawnTestWebServer(self, s_type, doc_root):
+    class Options(object):
+      log_to_console = True
+      server_type = s_type
+      port = self.server.test_server_port
+      data_dir = doc_root or 'chrome/test/data'
+      file_root_url = '/files/'
+      cert = False
+      policy_keys = None
+      policy_user = None
+      startup_pipe = None
+    options = Options()
+    logging.info('Listening on %d, type %d, data_dir %s' % (options.port,
+        options.server_type, options.data_dir))
+    testserver.main(options, None, server_list=_test_servers)
+    logging.info('Test-server has died.')
+
+
+class SpawningServer(object):
+  """The class used to start/stop a http server.
+  """
+
+  def __init__(self, test_server_spawner_port, test_server_port):
+    logging.info('Creating new spawner %d', test_server_spawner_port)
+    self.server = testserver.StoppableHTTPServer(('', test_server_spawner_port),
+                                                 SpawningServerRequestHandler)
+    self.port = test_server_spawner_port
+    self.server.test_server_port = test_server_port
+
+  def Listen(self):
+    logging.info('Starting test server spawner')
+    self.server.serve_forever()
+
+  def Start(self):
+    listener_thread = threading.Thread(target=self.Listen)
+    listener_thread.setDaemon(True)
+    listener_thread.start()
+    time.sleep(1)
+
+  def Stop(self):
+    self.server.Stop()

build/android/flag_changer.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+# Location where chrome reads command line flags from
+CHROME_COMMAND_FILE = '/data/local/chrome-command-line'
+
+
+class FlagChanger(object):
+  """Temporarily changes the flags Chrome runs with."""
+
+  def __init__(self, android_cmd):
+    self._android_cmd = android_cmd
+    self._old_flags = None
+
+  def Set(self, flags, append=False):
+    """Sets the command line flags used when chrome is started.
+
+    Args:
+      flags: A list of flags to set, eg. ['--single-process'].
+      append: Whether to append to existing flags or overwrite them.
+    """
+    if flags:
+      assert flags[0] != 'chrome'
+
+    if not self._old_flags:
+      self._old_flags = self._android_cmd.GetFileContents(CHROME_COMMAND_FILE)
+      if self._old_flags:
+        self._old_flags = self._old_flags[0].strip()
+
+    if append and self._old_flags:
+      # Avoid appending flags that are already present.
+      new_flags = filter(lambda flag: self._old_flags.find(flag) == -1, flags)
+      self._android_cmd.SetFileContents(CHROME_COMMAND_FILE,
+                                        self._old_flags + ' ' +
+                                        ' '.join(new_flags))
+    else:
+      self._android_cmd.SetFileContents(CHROME_COMMAND_FILE,
+                                        'chrome ' + ' '.join(flags))
+
+  def Restore(self):
+    """Restores the flags to their original state."""
+    if self._old_flags == None:
+      return  # Set() was never called.
+    elif self._old_flags:
+      self._android_cmd.SetFileContents(CHROME_COMMAND_FILE, self._old_flags)
+    else:
+      self._android_cmd.RunShellCommand('rm ' + CHROME_COMMAND_FILE)

build/android/gtest_filter/base_unittests_disabled

+# List of suppressions
+#
+# Automatically generated by run_tests.py
+RTLTest.GetTextDirection
+ReadOnlyFileUtilTest.ContentsEqual
+ReadOnlyFileUtilTest.TextContentsEqual
+SharedMemoryTest.OpenExclusive
+StackTrace.DebugPrintBacktrace

build/android/gtest_filter/base_unittests_emulator_additional_disabled

+# Addtional list of suppressions from emulator
+#
+# Automatically generated by run_tests.py
+PathServiceTest.Get
+SharedMemoryTest.OpenClose
+StringPrintfTest.StringAppendfInt
+StringPrintfTest.StringAppendfString
+StringPrintfTest.StringPrintfBounds
+StringPrintfTest.StringPrintfMisc
+VerifyPathControlledByUserTest.Symlinks

build/android/lighttpd_server.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Provides a convenient wrapper for spawning a test lighttpd instance.
+
+Usage:
+  lighttpd_server PATH_TO_DOC_ROOT
+"""
+
+import codecs
+import contextlib
+import httplib
+import os
+import pexpect
+import random
+import shutil
+import socket
+import sys
+import tempfile
+
+
+class LighttpdServer(object):
+  """Wraps lighttpd server, providing robust startup.
+
+  Args:
+    document_root: Path to root of this server's hosted files.
+    port: TCP port on the _host_ machine that the server will listen on. If
+        ommitted it will attempt to use 9000, or if unavailable it will find
+        a free port from 8001 - 8999.
+    lighttpd_path, lighttpd_module_path: Optional paths to lighttpd binaries.
+    base_config_path: If supplied this file will replace the built-in default
+        lighttpd config file.
+    extra_config_contents: If specified, this string will be appended to the
+        base config (default built-in, or from base_config_path).
+    config_path, error_log, access_log: Optional paths where the class should
+        place temprary files for this session.
+  """
+
+  def __init__(self, document_root, port=None,
+               lighttpd_path=None, lighttpd_module_path=None,
+               base_config_path=None, extra_config_contents=None,
+               config_path=None, error_log=None, access_log=None):
+    self.temp_dir = tempfile.mkdtemp(prefix='lighttpd_for_chrome_android')
+    self.document_root = os.path.abspath(document_root)
+    self.fixed_port = port
+    self.port = port or 9000
+    self.server_tag = 'LightTPD ' + str(random.randint(111111, 999999))
+    self.lighttpd_path = lighttpd_path or '/usr/sbin/lighttpd'
+    self.lighttpd_module_path = lighttpd_module_path or '/usr/lib/lighttpd'
+    self.base_config_path = base_config_path
+    self.extra_config_contents = extra_config_contents
+    self.config_path = config_path or self._Mktmp('config')
+    self.error_log = error_log or self._Mktmp('error_log')
+    self.access_log = access_log or self._Mktmp('access_log')
+    self.pid_file = self._Mktmp('pid_file')
+    self.process = None
+
+  def _Mktmp(self, name):
+    return os.path.join(self.temp_dir, name)
+
+  def _GetRandomPort(self):
+    # Ports 8001-8004 are reserved for other test servers. Ensure we don't
+    # collide with them.
+    return random.randint(8005, 8999)
+
+  def StartupHttpServer(self):
+    """Starts up a http server with specified document root and port."""
+    # Currently we use lighttpd as http sever in test.
+    while True:
+      if self.base_config_path:
+        # Read the config
+        with codecs.open(self.base_config_path, 'r', 'utf-8') as f:
+          config_contents = f.read()
+      else:
+        config_contents = self._GetDefaultBaseConfig()
+      if self.extra_config_contents:
+        config_contents += self.extra_config_contents
+      # Write out the config, filling in placeholders from the members of |self|
+      with codecs.open(self.config_path, 'w', 'utf-8') as f:
+        f.write(config_contents % self.__dict__)
+      if (not os.path.exists(self.lighttpd_path) or
+          not os.access(self.lighttpd_path, os.X_OK)):
+        raise EnvironmentError(
+            'Could not find lighttpd at %s.\n'
+            'It may need to be installed (e.g. sudo apt-get install lighttpd)'
+            % self.lighttpd_path)
+      self.process = pexpect.spawn(self.lighttpd_path,
+                                   ['-D', '-f', self.config_path,
+                                    '-m', self.lighttpd_module_path],
+                                   cwd=self.temp_dir)
+      client_error, server_error = self._TestServerConnection()
+      if not client_error:
+        assert int(open(self.pid_file, 'r').read()) == self.process.pid
+        break
+      self.process.close()
+
+      if self.fixed_port or not 'in use' in server_error:
+        print 'Client error:', client_error
+        print 'Server error:', server_error
+        return False
+      self.port = self._GetRandomPort()
+    return True
+
+  def ShutdownHttpServer(self):
+    """Shuts down our lighttpd processes."""
+    if self.process:
+      self.process.terminate()
+    shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+  def _TestServerConnection(self):
+    # Wait for server to start
+    server_msg = ''
+    for timeout in xrange(1, 5):
+      client_error = None
+      try:
+        with contextlib.closing(httplib.HTTPConnection(
+            '127.0.0.1', self.port, timeout=timeout)) as http:
+          http.set_debuglevel(timeout > 3)
+          http.request('HEAD', '/')
+          r = http.getresponse()
+          r.read()
+          if (r.status == 200 and r.reason == 'OK' and
+              r.getheader('Server') == self.server_tag):
+            return (None, server_msg)
+          client_error = ('Bad response: %s %s version %s\n  ' %
+                          (r.status, r.reason, r.version) +
+                          '\n  '.join([': '.join(h) for h in r.getheaders()]))
+      except (httplib.HTTPException, socket.error) as client_error:
+        pass  # Probably too quick connecting: try again
+      # Check for server startup error messages
+      ix = self.process.expect([pexpect.TIMEOUT, pexpect.EOF, '.+'],
+                               timeout=timeout)
+      if ix == 2:  # stdout spew from the server
+        server_msg += self.process.match.group(0)
+      elif ix == 1:  # EOF -- server has quit so giveup.
+        client_error = client_error or 'Server exited'
+        break
+    return (client_error or 'Timeout', server_msg)
+
+  def _GetDefaultBaseConfig(self):
+    return """server.tag                  = "%(server_tag)s"
+server.modules              = ( "mod_access",
+                                "mod_accesslog",
+                                "mod_alias",
+                                "mod_cgi",
+                                "mod_rewrite" )
+
+# default document root required
+#server.document-root = "."
+
+# files to check for if .../ is requested
+index-file.names            = ( "index.php", "index.pl", "index.cgi",
+                                "index.html", "index.htm", "default.htm" )
+# mimetype mapping
+mimetype.assign             = (
+  ".gif"          =>      "image/gif",
+  ".jpg"          =>      "image/jpeg",
+  ".jpeg"         =>      "image/jpeg",
+  ".png"          =>      "image/png",
+  ".svg"          =>      "image/svg+xml",
+  ".css"          =>      "text/css",
+  ".html"         =>      "text/html",
+  ".htm"          =>      "text/html",
+  ".xhtml"        =>      "application/xhtml+xml",
+  ".xhtmlmp"      =>      "application/vnd.wap.xhtml+xml",
+  ".js"           =>      "application/x-javascript",
+  ".log"          =>      "text/plain",
+  ".conf"         =>      "text/plain",
+  ".text"         =>      "text/plain",
+  ".txt"          =>      "text/plain",
+  ".dtd"          =>      "text/xml",
+  ".xml"          =>      "text/xml",
+  ".manifest"     =>      "text/cache-manifest",
+ )
+
+# Use the "Content-Type" extended attribute to obtain mime type if possible
+mimetype.use-xattr          = "enable"
+
+##
+# which extensions should not be handle via static-file transfer
+#
+# .php, .pl, .fcgi are most often handled by mod_fastcgi or mod_cgi
+static-file.exclude-extensions = ( ".php", ".pl", ".cgi" )
+
+server.bind = "127.0.0.1"
+server.port = %(port)s
+
+## virtual directory listings
+dir-listing.activate        = "enable"
+#dir-listing.encoding       = "iso-8859-2"
+#dir-listing.external-css   = "style/oldstyle.css"
+
+## enable debugging
+#debug.log-request-header   = "enable"
+#debug.log-response-header  = "enable"
+#debug.log-request-handling = "enable"
+#debug.log-file-not-found   = "enable"
+
+#### SSL engine
+#ssl.engine                 = "enable"
+#ssl.pemfile                = "server.pem"
+
+# Autogenerated test-specific config follows.
+
+cgi.assign = ( ".cgi"  => "/usr/bin/env",
+               ".pl"   => "/usr/bin/env",
+               ".asis" => "/bin/cat",
+               ".php"  => "/usr/bin/php-cgi" )
+
+server.errorlog = "%(error_log)s"
+accesslog.filename = "%(access_log)s"
+server.upload-dirs = ( "/tmp" )
+server.pid-file = "%(pid_file)s"
+server.document-root = "%(document_root)s"
+
+"""
+
+
+def main(argv):
+  server = LighttpdServer(*argv[1:])
+  try:
+    if server.StartupHttpServer():
+      raw_input('Server running at http://127.0.0.1:%s -'
+                ' press Enter to exit it.' % server.port)
+    else:
+      print 'Server exit code:', server.process.exitstatus
+  finally:
+    server.ShutdownHttpServer()
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

build/android/run_tests.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Runs all the native unit tests.
+
+1. Copy over test binary to /data/local on device.
+2. Resources: chrome/unit_tests requires resources (chrome.pak and en-US.pak)
+   to be deployed to the device (in /data/local/tmp).
+3. Environment:
+3.1. chrome/unit_tests requires (via chrome_paths.cc) a directory named:
+     /data/local/tmp/chrome/test/data
+3.2. page_cycler_tests have following requirements,
+3.2.1  the following data on host:
+       <chrome_src_dir>/tools/page_cycler
+       <chrome_src_dir>/data/page_cycler
+3.2.2. two data directories to store above test data on device named:
+       /data/local/tmp/tools/ (for database perf test)
+       /data/local/tmp/data/ (for other perf tests)
+3.2.3. a http server to serve http perf tests.
+       The http root is host's <chrome_src_dir>/data/page_cycler/, port 8000.
+3.2.4  a tool named forwarder is also required to run on device to
+       forward the http request/response between host and device.
+3.2.5  Chrome is installed on device.
+4. Run the binary in the device and stream the log to the host.
+4.1. Optionally, filter specific tests.
+4.2. Optionally, rebaseline: run the available tests and update the
+     suppressions file for failures.
+4.3. If we're running a single test suite and we have multiple devices
+     connected, we'll shard the tests.
+5. Clean up the device.
+
+Suppressions:
+
+Individual tests in a test binary can be suppressed by listing it in
+the gtest_filter directory in a file of the same name as the test binary,
+one test per line. Here is an example:
+
+  $ cat gtest_filter/base_unittests_disabled
+  DataPackTest.Load
+  ReadOnlyFileUtilTest.ContentsEqual
+
+This file is generated by the tests running on devices. If running on emulator,
+additonal filter file which lists the tests only failed in emulator will be
+loaded. We don't care about the rare testcases which succeeded on emuatlor, but
+failed on device.
+"""
+
+import logging
+import os
+import re
+import sys
+
+import android_commands
+import cmd_helper
+import debug_info
+import emulator
+import run_tests_helper
+from single_test_runner import SingleTestRunner
+from test_package_executable import TestPackageExecutable
+from test_result import BaseTestResult, TestResults
+
+_TEST_SUITES = ['base_unittests',]
+
+def RunTests(device, test_suite, gtest_filter, test_arguments, rebaseline,
+             timeout, performance_test, cleanup_test_files, tool,
+             log_dump_name):
+  """Runs the tests.
+
+  Args:
+    device: Device to run the tests.
+    test_suite: A specific test suite to run, empty to run all.
+    gtest_filter: A gtest_filter flag.
+    test_arguments: Additional arguments to pass to the test binary.
+    rebaseline: Whether or not to run tests in isolation and update the filter.
+    timeout: Timeout for each test.
+    performance_test: Whether or not performance test(s).
+    cleanup_test_files: Whether or not to cleanup test files on device.
+    tool: Name of the Valgrind tool.
+    log_dump_name: Name of log dump file.
+
+  Returns:
+    A TestResults object.
+  """
+  results = []
+
+  if test_suite:
+    global _TEST_SUITES
+    if not os.path.exists(test_suite):
+      logging.critical('Unrecognized test suite, supported: %s' %
+                       _TEST_SUITES)
+      if test_suite in _TEST_SUITES:
+        logging.critical('(Remember to include the path: out/Release/%s)',
+                         test_suite)
+      return TestResults.FromOkAndFailed([], [BaseTestResult(test_suite, '')])
+    _TEST_SUITES = [test_suite]
+  else:
+    # If not specified, assume the test suites are in out/Release
+    test_suite_dir = os.path.abspath(os.path.join(run_tests_helper.CHROME_DIR,
+        'out', 'Release'))
+    _TEST_SUITES = [os.path.join(test_suite_dir, t) for t in _TEST_SUITES]
+  debug_info_list = []
+  for t in _TEST_SUITES:
+    test = SingleTestRunner(device, t, gtest_filter, test_arguments,
+                            timeout, rebaseline, performance_test,
+                            cleanup_test_files, tool, not not log_dump_name)
+    test.RunTests()
+    results += [test.test_results]
+    # Collect debug info.
+    debug_info_list += [test.dump_debug_info]
+    if rebaseline:
+      test.UpdateFilter(test.test_results.failed)
+    elif test.test_results.failed:
+      # Stop running test if encountering failed test.
+      test.test_results.LogFull()
+      break
+  # Zip all debug info outputs into a file named by log_dump_name.
+  debug_info.GTestDebugInfo.ZipAndCleanResults(
+      os.path.join(run_tests_helper.CHROME_DIR, 'out', 'Release',
+          'debug_info_dumps'),
+      log_dump_name, [d for d in debug_info_list if d])
+  return TestResults.FromTestResults(results)
+
+def Dispatch(options):
+  """Dispatches the tests, sharding if possible.
+
+  If options.use_emulator is True, all tests will be run in a new emulator
+  instance.
+
+  Args:
+    options: options for running the tests.
+
+  Returns:
+    0 if successful, number of failing tests otherwise.
+  """
+  if options.test_suite == 'help':
+    ListTestSuites()
+    return 0
+  buildbot_emulator = None
+  attached_devices = []
+
+  if options.use_emulator:
+    buildbot_emulator = emulator.Emulator()
+    buildbot_emulator.Launch()
+    attached_devices.append(buildbot_emulator.device)
+  else:
+    attached_devices = android_commands.GetAttachedDevices()
+
+  if not attached_devices:
+    logging.critical('A device must be attached and online.')
+    return 1
+
+  test_results = RunTests(attached_devices[0], options.test_suite,
+                          options.gtest_filter, options.test_arguments,
+                          options.rebaseline, options.timeout,
+                          options.performance_test,
+                          options.cleanup_test_files, options.tool,
+                          options.log_dump)
+  if buildbot_emulator:
+    buildbot_emulator.Shutdown()
+  return len(test_results.failed)
+
+def ListTestSuites():
+  """Display a list of available test suites
+  """
+  print 'Available test suites are:'
+  for test_suite in _TEST_SUITES:
+    print test_suite
+
+
+def main(argv):
+  option_parser = run_tests_helper.CreateTestRunnerOptionParser(None,
+      default_timeout=0)
+  option_parser.add_option('-s', dest='test_suite',
+                           help='Executable name of the test suite to run '
+                           '(use -s help to list them)')
+  option_parser.add_option('-r', dest='rebaseline',
+                           help='Rebaseline and update *testsuite_disabled',
+                           action='store_true',
+                           default=False)
+  option_parser.add_option('-f', dest='gtest_filter',
+                           help='gtest filter')
+  option_parser.add_option('-a', '--test_arguments', dest='test_arguments',
+                           help='Additional arguments to pass to the test')
+  option_parser.add_option('-p', dest='performance_test',
+                           help='Indicator of performance test',
+                           action='store_true',
+                           default=False)
+  option_parser.add_option('-L', dest='log_dump',
+                           help='file name of log dump, which will be put in'
+                           'subfolder debug_info_dumps under the same directory'
+                           'in where the test_suite exists.')
+  option_parser.add_option('-e', '--emulator', dest='use_emulator',
+                           help='Run tests in a new instance of emulator',
+                           action='store_true',
+                           default=False)
+  options, args = option_parser.parse_args(argv)
+  if len(args) > 1:
+    print 'Unknown argument:', args[1:]
+    option_parser.print_usage()
+    sys.exit(1)
+  run_tests_helper.SetLogLevel(options.verbose_count)
+  return Dispatch(options)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

build/android/run_tests_helper.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Helper functions common to native test runners."""
+
+import logging
+import optparse
+import os
+import subprocess
+import sys
+
+# TODO(michaelbai): Move constant definitions like below to a common file.
+FORWARDER_PATH = '/data/local/tmp/forwarder'
+
+CHROME_DIR = os.path.abspath(os.path.join(sys.path[0], '..', '..'))
+
+
+def IsRunningAsBuildbot():
+  """Returns True if we are currently running on buildbot; False otherwise."""
+  return bool(os.getenv('BUILDBOT_BUILDERNAME'))
+
+
+def ReportBuildbotLink(label, url):
+  """Adds a link with name |label| linking to |url| to current buildbot step.
+
+  Args:
+    label: A string with the name of the label.
+    url: A string of the URL.
+  """
+  if IsRunningAsBuildbot():
+    print '@@@STEP_LINK@%s@%s@@@' % (label, url)
+
+
+def ReportBuildbotMsg(msg):
+  """Appends |msg| to the current buildbot step text.
+
+  Args:
+    msg: String to be appended.
+  """
+  if IsRunningAsBuildbot():
+    print '@@@STEP_TEXT@%s@@@' % msg
+
+def ReportBuildbotError():
+  """Marks the current step as failed."""
+  if IsRunningAsBuildbot():
+    print '@@@STEP_FAILURE@@@'
+
+
+def GetExpectations(file_name):
+  """Returns a list of test names in the |file_name| test expectations file."""
+  if not file_name or not os.path.exists(file_name):
+    return []
+  return [x for x in [x.strip() for x in file(file_name).readlines()]
+          if x and x[0] != '#']
+
+
+def SetLogLevel(verbose_count):
+  """Sets log level as |verbose_count|."""
+  log_level = logging.WARNING  # Default.
+  if verbose_count == 1:
+    log_level = logging.INFO
+  elif verbose_count >= 2:
+    log_level = logging.DEBUG
+  logging.getLogger().setLevel(log_level)
+
+
+def CreateTestRunnerOptionParser(usage=None, default_timeout=60):
+  """Returns a new OptionParser with arguments applicable to all tests."""
+  option_parser = optparse.OptionParser(usage=usage)
+  option_parser.add_option('-t', dest='timeout',
+                           help='Timeout to wait for each test',
+                           type='int',
+                           default=default_timeout)
+  option_parser.add_option('-c', dest='cleanup_test_files',
+                           help='Cleanup test files on the device after run',
+                           action='store_true',
+                           default=False)
+  option_parser.add_option('-v',
+                           '--verbose',
+                           dest='verbose_count',
+                           default=0,
+                           action='count',
+                           help='Verbose level (multiple times for more)')
+  option_parser.add_option('--tool',
+                           dest='tool',
+                           help='Run the test under a tool '
+                           '(use --tool help to list them)')
+  return option_parser
+
+
+def ForwardDevicePorts(adb, ports, host_name='127.0.0.1'):
+  """Forwards a TCP port on the device back to the host.
+
+  Works like adb forward, but in reverse.
+
+  Args:
+    adb: Instance of AndroidCommands for talking to the device.
+    ports: A list of tuples (device_port, host_port) to forward.
+    host_name: Optional. Address to forward to, must be addressable from the
+               host machine. Usually this is omitted and loopback is used.
+
+  Returns:
+    subprocess instance connected to the forwarder process on the device.
+  """
+  adb.PushIfNeeded(
+      os.path.join(CHROME_DIR, 'out', 'Release', 'forwarder'), FORWARDER_PATH)
+  forward_string = ['%d:%d:%s' %
+                    (device, host, host_name) for device, host in ports]
+  logging.info("Forwarding ports: %s" % (forward_string))
+
+  return subprocess.Popen(
+      ['adb', '-s', adb._adb.GetSerialNumber(),
+       'shell', '%s -D %s' % (FORWARDER_PATH, ' '.join(forward_string))])
+
+
+def IsDevicePortUsed(adb, device_port):
+  """Checks whether the specified device port is used or not.
+
+  Args:
+    adb: Instance of AndroidCommands for talking to the device.
+    device_port: Port on device we want to check.
+
+  Returns:
+    True if the port on device is already used, otherwise returns False.
+  """
+  base_url = '127.0.0.1:%d' % device_port
+  netstat_results = adb.RunShellCommand('netstat')
+  for single_connect in netstat_results:
+    # Column 3 is the local address which we want to check with.
+    if single_connect.split()[3] == base_url:
+      return True
+  return False

build/android/test_package.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import logging
+import re
+import os
+import pexpect
+
+from perf_tests_helper import PrintPerfResult
+from test_result import BaseTestResult, TestResults
+from valgrind_tools import CreateTool
+
+
+# TODO(bulach): TestPackage, TestPackageExecutable and
+# TestPackageApk are a work in progress related to making the native tests
+# run as a NDK-app from an APK rather than a stand-alone executable.
+class TestPackage(object):
+  """A helper base class for both APK and stand-alone executables.
+
+  Args:
+    adb: ADB interface the tests are using.
+    device: Device to run the tests.
+    test_suite: A specific test suite to run, empty to run all.
+    timeout: Timeout for each test.
+    rebaseline: Whether or not to run tests in isolation and update the filter.
+    performance_test: Whether or not performance test(s).
+    cleanup_test_files: Whether or not to cleanup test files on device.
+    tool: Name of the Valgrind tool.
+    dump_debug_info: A debug_info object.
+  """
+
+  def __init__(self, adb, device, test_suite, timeout, rebaseline,
+               performance_test, cleanup_test_files, tool, dump_debug_info):
+    self.adb = adb
+    self.device = device
+    self.test_suite = os.path.splitext(test_suite)[0]
+    self.test_suite_basename = os.path.basename(self.test_suite)
+    self.test_suite_dirname = os.path.dirname(self.test_suite)
+    self.rebaseline = rebaseline
+    self._performance_test = performance_test
+    self.cleanup_test_files = cleanup_test_files
+    self.tool = CreateTool(tool, self.adb)
+    if timeout == 0:
+      if self.test_suite_basename == 'page_cycler_tests':
+        timeout = 900
+      else:
+        timeout = 60
+    self.timeout = timeout * self.tool.GetTimeoutScale()
+    self.dump_debug_info = dump_debug_info
+
+  def _BeginGetIOStats(self):
+    """Gets I/O statistics before running test.
+
+    Return:
+      Tuple of (I/O stats object, flag of ready to continue). When encountering
+      error, ready-to-continue flag is False, True otherwise. The I/O stats
+      object may be None if the test is not performance test.
+    """
+    initial_io_stats = None
+    # Try to get the disk I/O statistics for all performance tests.
+    if self._performance_test and not self.rebaseline:
+      initial_io_stats = self.adb.GetIoStats()
+      # Get rid of the noise introduced by launching Chrome for page cycler.
+      if self.test_suite_basename == 'page_cycler_tests':
+        try:
+          chrome_launch_done_re = re.compile(
+              re.escape('Finish waiting for browser launch!'))
+          self.adb.WaitForLogMatch(chrome_launch_done_re)
+          initial_io_stats = self.adb.GetIoStats()
+        except pexpect.TIMEOUT:
+          logging.error('Test terminated because Chrome launcher has no'
+                        'response after 120 second.')
+          return (None, False)
+        finally:
+          if self.dump_debug_info:
+            self.dump_debug_info.TakeScreenshot('_Launch_Chrome_')
+    return (initial_io_stats, True)
+
+  def _EndGetIOStats(self, initial_io_stats):
+    """Gets I/O statistics after running test and calcuate the I/O delta.
+
+    Args:
+      initial_io_stats: I/O stats object got from _BeginGetIOStats.
+
+    Return:
+      String for formated diso I/O statistics.
+    """
+    disk_io = ''
+    if self._performance_test and initial_io_stats:
+      final_io_stats = self.adb.GetIoStats()
+      for stat in final_io_stats:
+        disk_io += '\n' + PrintPerfResult(stat, stat,
+                                          [final_io_stats[stat] -
+                                           initial_io_stats[stat]],
+                                          stat.split('_')[1], True, False)
+      logging.info(disk_io)
+    return disk_io
+
+  def GetDisabledPrefixes(self):
+    return ['DISABLED_', 'FLAKY_', 'FAILS_']
+
+  def _ParseGTestListTests(self, all_tests):
+    ret = []
+    current = ''
+    disabled_prefixes = self.GetDisabledPrefixes()
+    for test in all_tests:
+      if not test:
+        continue
+      if test[0] != ' ':
+        current = test
+        continue
+      if 'YOU HAVE' in test:
+        break
+      test_name = test[2:]
+      if not any([test_name.startswith(x) for x in disabled_prefixes]):
+        ret += [current + test_name]
+    return ret
+
+  def _WatchTestOutput(self, p):
+    """Watches the test output.
+    Args:
+      p: the process generating output as created by pexpect.spawn.
+    """
+    ok_tests = []
+    failed_tests = []
+    re_run = re.compile('\[ RUN      \] ?(.*)\r\n')
+    re_fail = re.compile('\[  FAILED  \] ?(.*)\r\n')
+    re_ok = re.compile('\[       OK \] ?(.*)\r\n')
+    (io_stats_before, ready_to_continue) = self._BeginGetIOStats()
+    while ready_to_continue:
+      found = p.expect([re_run, pexpect.EOF], timeout=self.timeout)
+      if found == 1:  # matched pexpect.EOF
+        break
+      if self.dump_debug_info:
+        self.dump_debug_info.TakeScreenshot('_Test_Start_Run_')
+      full_test_name = p.match.group(1)
+      found = p.expect([re_ok, re_fail, pexpect.EOF, pexpect.TIMEOUT],
+                       timeout=self.timeout)
+      if found == 0:  # re_ok
+        ok_tests += [BaseTestResult(full_test_name.replace('\r', ''),
+                                    p.before)]
+        continue
+      failed_tests += [BaseTestResult(full_test_name.replace('\r', ''),
+                                      p.before)]
+      if found >= 2:
+        # The test crashed / bailed out (i.e., didn't print OK or FAIL).
+        if found == 3:  # pexpect.TIMEOUT
+          logging.error('Test terminated after %d second timeout.',
+                        self.timeout)
+        break
+    p.close()
+    if not self.rebaseline and ready_to_continue:
+      ok_tests += self._EndGetIOStats(io_stats_before)
+      ret_code = self._GetGTestReturnCode()
+      if ret_code:
+        failed_tests += [BaseTestResult('gtest exit code: %d' % ret_code,
+                                        'pexpect.before: %s'
+                                        '\npexpect.after: %s'
+                                        % (p.before,
+                                           p.after))]
+    return TestResults.FromOkAndFailed(ok_tests, failed_tests)

build/android/test_package_executable.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import logging
+import os
+import pexpect
+import shutil
+import sys
+import tempfile
+
+import cmd_helper
+from test_package import TestPackage
+
+
+class TestPackageExecutable(TestPackage):
+  """A helper class for running stand-alone executables."""
+
+  _TEST_RUNNER_RET_VAL_FILE = '/data/local/tmp/gtest_retval'
+
+  def __init__(self, adb, device, test_suite, timeout, rebaseline,
+               performance_test, cleanup_test_files, tool, dump_debug_info,
+               symbols_dir=None):
+    """
+    Args:
+      adb: ADB interface the tests are using.
+      device: Device to run the tests.
+      test_suite: A specific test suite to run, empty to run all.
+      timeout: Timeout for each test.
+      rebaseline: Whether or not to run tests in isolation and update the
+          filter.
+      performance_test: Whether or not performance test(s).
+      cleanup_test_files: Whether or not to cleanup test files on device.
+      tool: Name of the Valgrind tool.
+      dump_debug_info: A debug_info object.
+      symbols_dir: Directory to put the stripped binaries.
+    """
+    TestPackage.__init__(self, adb, device, test_suite, timeout,
+                         rebaseline, performance_test, cleanup_test_files,
+                         tool, dump_debug_info)
+    self.symbols_dir = symbols_dir
+
+  def _GetGTestReturnCode(self):
+    ret = None
+    ret_code_file = tempfile.NamedTemporaryFile()
+    try:
+      if not self.adb.Adb().Pull(
+          TestPackageExecutable._TEST_RUNNER_RET_VAL_FILE, ret_code_file.name):
+        logging.critical('Unable to pull gtest ret val file %s',
+                         ret_code_file.name)
+        raise ValueError
+      ret_code = file(ret_code_file.name).read()
+      ret = int(ret_code)
+    except ValueError:
+      logging.critical('Error reading gtest ret val file %s [%s]',
+                       ret_code_file.name, ret_code)
+      ret = 1
+    return ret
+
+  def _AddNativeCoverageExports(self):
+    # export GCOV_PREFIX set the path for native coverage results
+    # export GCOV_PREFIX_STRIP indicates how many initial directory
+    #                          names to strip off the hardwired absolute paths.
+    #                          This value is calculated in buildbot.sh and
+    #                          depends on where the tree is built.
+    # Ex: /usr/local/google/code/chrome will become
+    #     /code/chrome if GCOV_PREFIX_STRIP=3
+    try:
+      depth = os.environ['NATIVE_COVERAGE_DEPTH_STRIP']
+    except KeyError:
+      logging.info('NATIVE_COVERAGE_DEPTH_STRIP is not defined: '
+                   'No native coverage.')
+      return ''
+    export_string = 'export GCOV_PREFIX="/data/local/gcov"\n'
+    export_string += 'export GCOV_PREFIX_STRIP=%s\n' % depth
+    return export_string
+
+  def GetAllTests(self):
+    """Returns a list of all tests available in the test suite."""
+    all_tests = self.adb.RunShellCommand(
+        '/data/local/%s --gtest_list_tests' % self.test_suite_basename)
+    return self._ParseGTestListTests(all_tests)
+
+  def CreateTestRunnerScript(self, gtest_filter, test_arguments):
+    """Creates a test runner script and pushes to the device.
+
+    Args:
+      gtest_filter: A gtest_filter flag.
+      test_arguments: Additional arguments to pass to the test binary.
+    """
+    tool_wrapper = self.tool.GetTestWrapper()
+    sh_script_file = tempfile.NamedTemporaryFile()
+    # We need to capture the exit status from the script since adb shell won't
+    # propagate to us.
+    sh_script_file.write('cd /data/local\n'
+                         '%s'
+                         '%s /data/local/%s --gtest_filter=%s %s\n'
+                         'echo $? > %s' %
+                         (self._AddNativeCoverageExports(),
+                          tool_wrapper, self.test_suite_basename,
+                          gtest_filter, test_arguments,
+                          TestPackageExecutable._TEST_RUNNER_RET_VAL_FILE))
+    sh_script_file.flush()
+    cmd_helper.RunCmd(['chmod', '+x', sh_script_file.name])
+    self.adb.PushIfNeeded(sh_script_file.name,
+                          '/data/local/chrome_test_runner.sh')
+
+  def RunTestsAndListResults(self):
+    """Runs all the tests and checks for failures.
+
+    Returns:
+      A TestResults object.
+    """
+    args = ['adb', '-s', self.device, 'shell', 'sh',
+            '/data/local/chrome_test_runner.sh']
+    logging.info(args)
+    p = pexpect.spawn(args[0], args[1:], logfile=sys.stdout)
+    return self._WatchTestOutput(p)
+
+  def StripAndCopyExecutable(self):
+    """Strips and copies the executable to the device."""
+    if self.tool.NeedsDebugInfo():
+      target_name = self.test_suite
+    elif self.test_suite_basename == 'webkit_unit_tests':
+      # webkit_unit_tests has been stripped in build step.
+      target_name = self.test_suite
+    else:
+      target_name = self.test_suite + '_' + self.device + '_stripped'
+      should_strip = True
+      if os.path.isfile(target_name):
+        logging.info('Found target file %s' % target_name)
+        target_mtime = os.stat(target_name).st_mtime
+        source_mtime = os.stat(self.test_suite).st_mtime
+        if target_mtime > source_mtime:
+          logging.info('Target mtime (%d) is newer than source (%d), assuming '
+                       'no change.' % (target_mtime, source_mtime))
+          should_strip = False
+
+      if should_strip:
+        logging.info('Did not find up-to-date stripped binary. Generating a '
+                     'new one (%s).' % target_name)
+        # Whenever we generate a stripped binary, copy to the symbols dir. If we
+        # aren't stripping a new binary, assume it's there.
+        if self.symbols_dir:
+          if not os.path.exists(self.symbols_dir):
+            os.makedirs(self.symbols_dir)
+          shutil.copy(self.test_suite, self.symbols_dir)
+        strip = os.environ['STRIP']
+        cmd_helper.RunCmd([strip, self.test_suite, '-o', target_name])
+    test_binary = '/data/local/' + self.test_suite_basename
+    self.adb.PushIfNeeded(target_name, test_binary)

build/android/test_result.py

+#!/usr/bin/python
+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import logging
+
+
+# Language values match constants in Sponge protocol buffer (sponge.proto).
+JAVA = 5
+PYTHON = 7
+
+
+class BaseTestResult(object):
+  """A single result from a unit test."""
+
+  def __init__(self, name, log):
+    self.name = name
+    self.log = log
+
+
+class SingleTestResult(BaseTestResult):
+  """Result information for a single test.
+
+  Args:
+    full_name: Full name of the test.
+    start_date: Date in milliseconds when the test began running.
+    dur: Duration of the test run in milliseconds.
+    lang: Language of the test (JAVA or PYTHON).
+    log: An optional string listing any errors.
+    error: A tuple of a short error message and a longer version used by Sponge
+    if test resulted in a fail or error.  An empty tuple implies a pass.
+  """
+
+  def __init__(self, full_name, start_date, dur, lang, log='', error=()):
+    BaseTestResult.__init__(self, full_name, log)
+    name_pieces = full_name.rsplit('#')
+    if len(name_pieces) > 0:
+      self.test_name = name_pieces[1]
+      self.class_name = name_pieces[0]
+    else:
+      self.class_name = full_name
+      self.test_name = full_name
+    self.start_date = start_date
+    self.dur = dur
+    self.error = error
+    self.lang = lang
+
+
+class TestResults(object):
+  """Results of a test run."""
+
+  def __init__(self):
+    self.ok = []
+    self.failed = []
+    self.crashed = []
+    self.unknown = []
+    self.disabled = []
+    self.unexpected_pass = []
+
+  @staticmethod
+  def FromOkAndFailed(ok, failed):
+    ret = TestResults()
+    ret.ok = ok
+    ret.failed = failed
+    return ret
+
+  @staticmethod
+  def FromTestResults(results):
+    """Combines a list of results in a single TestResults object."""
+    ret = TestResults()
+    for t in results:
+      ret.ok += t.ok
+      ret.failed += t.failed
+      ret.crashed += t.crashed
+      ret.unknown += t.unknown
+      ret.disabled += t.disabled
+      ret.unexpected_pass += t.unexpected_pass
+    return ret
+
+  def _Log(self, sorted_list):
+    for t in sorted_list:
+      logging.critical(t.name)
+      if t.log:
+        logging.critical(t.log)
+
+  def GetAllBroken(self):
+    """Returns the all broken tests including failed, crashed, unknown."""
+    return self.failed + self.crashed + self.unknown
+
+  def LogFull(self):
+    """Output all broken tests or 'passed' if none broken"""
+    logging.critical('*' * 80)
+    logging.critical('Final result')
+    if self.failed:
+      logging.critical('Failed:')
+      self._Log(sorted(self.failed))
+    if self.crashed:
+      logging.critical('Crashed:')
+      self._Log(sorted(self.crashed))
+    if self.unknown:
+      logging.critical('Unknown:')
+      self._Log(sorted(self.unknown))
+    if not self.GetAllBroken():
+      logging.critical('Passed')
+    logging.critical('*' * 80)