Make run_cts.py work with multiple apk files

Bug: 678127
Change-Id: I44c0ac8fecfde34ecf67ad0f74fe1d84bf7b2594
Reviewed-on: https://chromium-review.googlesource.com/924248
Commit-Queue: Andrew Luo <aluo@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Reviewed-by: Richard Coles <torne@chromium.org>
Cr-Commit-Position: refs/heads/master@{#552137}

android_webview/tools/cts_config/expected_failure_on_bot.json

@@ -40,5 +40,11 @@
       "name": "testServiceWorkerClientInterceptCallback",
       "_bug_id": "crbug.com/675809"
     }
+  ],
+  "android.webkit.cts.WebChromeClientTest": [
+    {
+      "name": "testOnJsBeforeUnloadIsCalled",
+      "_bug_id": "crbug.com/757007"
+    }
   ]
 }

android_webview/tools/cts_config/webview_cts_gcs_path.json

 {
   "arm64": {
     "L": {
-        "filename": "arm64/L/android-cts-arm_64-3500656.zip",
-        "apk": "arm64/L/3500656/CtsWebkitTestCases.apk",
-        "_origin": "aosp-lollipop-mr1-cts-dev@3500656"
+        "filename": "arm64/L/android-cts-arm_64-4607260.zip",
+        "apkdir": "arm64/L/4607260/",
+        "tests": [
+                {"android-cts/repository/testcases/CtsWebkitTestCases.apk": ""},
+                {"android-cts/repository/testcases/CtsWidgetTestCases.apk": "android.widget.cts.RemoteViewsActivityTest#testWebView"}
+        ],
+        "_origin": "aosp-lollipop-mr1-cts-dev@4607260"
     },
     "M": {
-        "filename": "arm64/M/android-cts-arm_64-3505727.zip",
-        "apk": "arm64/M/3505727/CtsWebkitTestCases.apk",
-        "_origin": "aosp-marshmallow-cts-dev@3505727"
+        "filename": "arm64/M/android-cts-arm_64-4607285.zip",
+        "apkdir": "arm64/M/4607285/",
+        "tests": [
+                {"android-cts/repository/testcases/CtsWebkitTestCases.apk": ""},
+                {"android-cts/repository/testcases/CtsWidgetTestCases.apk": "android.widget.cts.RemoteViewsActivityTest#testWebView"}
+        ],
+        "_origin": "aosp-marshmallow-cts-dev@4607285"
     },
     "N": {
-        "filename": "arm64/N/android-cts-arm_64-3367285.zip",
-        "apk": "arm64/N/3367285/CtsWebkitTestCases.apk",
-        "_origin": "aosp-nougat-cts-release@3367285"
+        "filename": "arm64/N/android-cts-arm_64-4606956.zip",
+        "apkdir": "arm64/N/4606956/",
+        "tests": [
+                {"android-cts/testcases/CtsWebkitTestCases.apk": ""},
+                {"android-cts/testcases/CtsWidgetTestCases.apk": "android.widget.cts.RemoteViewsActivityTest#testWebView"}
+        ],
+        "_origin": "aosp-nougat-cts-release@4606956"
     }
   }
 }

android_webview/tools/run_cts.py

@@ -12,6 +12,7 @@ import os
 import shutil
 import sys
 import tempfile
+import zipfile
 
 
 sys.path.append(os.path.join(
@@ -23,6 +24,9 @@ sys.path.append(os.path.join(
     os.path.dirname(__file__), os.pardir, os.pardir, 'build'))
 import find_depot_tools  # pylint: disable=import-error
 
+# cts test archives for all platforms are stored in this bucket
+# contents need to be updated if there is an important fix to any of
+# the tests
 _CTS_BUCKET = 'gs://chromium-cts'
 
 _GSUTIL_PATH = os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py')
@@ -36,19 +40,25 @@ _WEBVIEW_CTS_GCS_PATH_FILE = os.path.join(
     os.path.dirname(__file__), 'cts_config', 'webview_cts_gcs_path.json')
 
 
-def GetCtsPath(arch, platform):
-  """Gets relative path the CTS APK is stored at."""
+def GetCtsInfo(arch, platform, item):
+  """Gets contents of CTS Info for arch and platform.
+
+  See _WEBVIEW_CTS_GCS_PATH_FILE
+  """
   with open(_WEBVIEW_CTS_GCS_PATH_FILE) as f:
     cts_gcs_path_info = json.load(f)
   try:
-    return cts_gcs_path_info[arch][platform]['apk']
+    return cts_gcs_path_info[arch][platform][item]
   except KeyError:
-    raise Exception('No CTS test available for arch:%s, android:%s' %
-                    (arch, platform))
+    raise Exception('No %s info available for arch:%s, android:%s' %
+                    (item, arch, platform))
 
 
 def GetExpectedFailures():
-  """Gets list of tests expected to fail."""
+  """Gets list of tests expected to fail in <class>#<method> format.
+
+  See _EXPECTED_FAILURES_FILE
+  """
   with open(_EXPECTED_FAILURES_FILE) as f:
     expected_failures_info = json.load(f)
   expected_failures = []
@@ -58,38 +68,133 @@ def GetExpectedFailures():
   return expected_failures
 
 
-def DownloadAndRunCTS(args, test_runner_args):
+def RunCTS(test_runner_args, local_cts_dir, apk, test_filter,
+           skip_expected_failures=True, json_results_file=None):
+  """Run tests in apk using test_runner script at _TEST_RUNNER_PATH.
+
+  Returns the script result code,
+  tests expected to fail will be skipped unless skip_expected_failures
+  is set to False, test results will be stored in
+  the json_results_file file if specified
+  """
+  local_test_runner_args = test_runner_args + ['--test-apk',
+                                               os.path.join(local_cts_dir, apk)]
+
+  # TODO(mikecase): This doesn't work at all with the
+  # --gtest-filter test runner option currently. The
+  # filter options will just override eachother.
+  if skip_expected_failures:
+    local_test_runner_args += ['-f=-%s' % ':'.join(GetExpectedFailures())]
+  # The preferred method is to specify test filters per release in
+  # the CTS_GCS path file.  It will override any
+  # previous filters, including ones in expected failures
+  # file.
+  if test_filter:
+    local_test_runner_args += ['-f=' + test_filter]
+  if json_results_file:
+    local_test_runner_args += ['--json-results-file=%s' %
+                               json_results_file]
+  return cmd_helper.RunCmd(
+      [_TEST_RUNNER_PATH, 'instrumentation'] + local_test_runner_args)
+
+
+def MergeTestResults(existing_results_json, additional_results_json):
+  """Appends results in additional_results_json to existing_results_json."""
+  for k, v in additional_results_json.iteritems():
+    if k not in existing_results_json:
+      existing_results_json[k] = v
+    else:
+      if type(v) != type(existing_results_json[k]):
+        raise NotImplementedError(
+            "Can't merge results field %s of different types" % v)
+      if type(v) is dict:
+        existing_results_json[k].update(v)
+      elif type(v) is list:
+        existing_results_json[k].extend(v)
+      else:
+        raise NotImplementedError(
+            "Can't merge results field %s that is not a list or dict" % v)
+
+
+def DownloadAndExtractCTSZip(args):
+  """Download and extract the CTS tests from _CTS_BUCKET.
+
+  Downloads the CTS zip file from _CTS_BUCKET and extract contents to
+  apk_dir if specified, or a new temporary directory if not.
+  Returns following tuple (local_cts_dir, base_cts_dir, delete_cts_dir):
+    local_cts_dir - CTS extraction location for current arch and platform
+    base_cts_dir - Root directory for all the arches and platforms
+    delete_cts_dir - Set if the base_cts_dir was created as a temporary
+    directory
+  """
   base_cts_dir = None
   delete_cts_dir = False
-  try:
-    relative_cts_path = GetCtsPath(args.arch, args.platform)
+  relative_cts_zip_path = GetCtsInfo(args.arch, args.platform, 'filename')
 
-    if args.apk_dir:
-      base_cts_dir = args.apk_dir
-    else:
-      base_cts_dir = tempfile.mkdtemp()
-      delete_cts_dir = True
-
-    local_cts_path = os.path.join(base_cts_dir, relative_cts_path)
-    google_storage_cts_path = '%s/%s' % (_CTS_BUCKET, relative_cts_path)
-
-    # Download CTS APK if needed.
-    if not os.path.exists(local_cts_path):
-      if cmd_helper.RunCmd(
-          [_GSUTIL_PATH, 'cp', google_storage_cts_path, local_cts_path]):
-        raise Exception('Error downloading CTS from Google Storage.')
-
-    test_runner_args += ['--test-apk', local_cts_path]
-    # TODO(mikecase): This doesn't work at all with the
-    # --gtest-filter test runner option currently. The
-    # filter options will just override eachother.
-    if args.skip_expected_failures:
-      test_runner_args += ['-f=-%s' % ':'.join(GetExpectedFailures())]
-    return cmd_helper.RunCmd(
-        [_TEST_RUNNER_PATH, 'instrumentation'] + test_runner_args)
+  if args.apk_dir:
+    base_cts_dir = args.apk_dir
+  else:
+    base_cts_dir = tempfile.mkdtemp()
+    delete_cts_dir = True
+
+  local_cts_zip_path = os.path.join(base_cts_dir, relative_cts_zip_path)
+  google_storage_cts_zip_path = '%s/%s' % (
+      _CTS_BUCKET, os.path.join(relative_cts_zip_path))
+
+  # Download CTS APK if needed.
+  if not os.path.exists(local_cts_zip_path):
+    if cmd_helper.RunCmd([_GSUTIL_PATH, 'cp', google_storage_cts_zip_path,
+                          local_cts_zip_path]):
+      raise Exception('Error downloading CTS from Google Storage.')
+
+  local_cts_dir = os.path.join(base_cts_dir,
+                               GetCtsInfo(args.arch, args.platform, 'apkdir'))
+  zf = zipfile.ZipFile(local_cts_zip_path, 'r')
+  zf.extractall(local_cts_dir)
+  return (local_cts_dir, base_cts_dir, delete_cts_dir)
+
+
+def DownloadAndRunCTS(args, test_runner_args):
+  """Run CTS tests downloaded from _CTS_BUCKET.
+
+  Downloads CTS tests from bucket, runs them for the
+  specified platform+arch, then creates a single
+  results json file (if specified)
+  Returns 0 if all tests passed, otherwise
+  returns the failure code of the last failing
+  test.
+  """
+  local_cts_dir, base_cts_dir, delete_cts_dir = DownloadAndExtractCTSZip(args)
+  cts_result = 0
+  json_results_file = args.json_results_file
+  try:
+    cts_tests_info = GetCtsInfo(args.arch, args.platform, 'tests')
+    cts_results_json = {}
+    for cts_tests_item in cts_tests_info:
+      for relative_apk_path, test_filter in cts_tests_item.iteritems():
+        iteration_cts_result = 0
+        if json_results_file:
+          with tempfile.NamedTemporaryFile() as iteration_json_file:
+            iteration_cts_result = RunCTS(test_runner_args, local_cts_dir,
+                                          relative_apk_path, test_filter,
+                                          args.skip_expected_failures,
+                                          iteration_json_file.name)
+            with open(iteration_json_file.name) as f:
+              additional_results_json = json.load(f)
+              MergeTestResults(cts_results_json, additional_results_json)
+        else:
+          iteration_cts_result = RunCTS(test_runner_args, local_cts_dir,
+                                        relative_apk_path, test_filter,
+                                        args.skip_expected_failures)
+        if iteration_cts_result:
+          cts_result = iteration_cts_result
+    if json_results_file:
+      with open(json_results_file, 'w') as f:
+        json.dump(cts_results_json, f, indent=2)
   finally:
     if delete_cts_dir and base_cts_dir:
       shutil.rmtree(base_cts_dir)
+  return cts_result
 
 
 def main():
@@ -113,6 +218,13 @@ def main():
       help='Directory to load/save CTS APKs. Will try to load CTS APK '
            'from this directory before downloading from Google Storage '
            'and will then cache APK here.')
+  parser.add_argument(
+      '--test-launcher-summary-output',
+      '--json-results-file',
+      dest='json_results_file', type=os.path.realpath,
+      help='If set, will dump results in JSON form to the specified file. '
+           'Note that this will also trigger saving per-test logcats to '
+           'logdog.')
 
   args, test_runner_args = parser.parse_known_args()
   devil_chromium.Initialize()