Improve webview_cts_gcs_path.json format

Bug: 902958
Change-Id: I1c8a6fbc21c871ced824c2b387196d2eae856728
Reviewed-on: https://chromium-review.googlesource.com/c/1324991Reviewed-by: Bo <boliu@chromium.org>
Commit-Queue: Andrew Luo <aluo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#606346}

android_webview/tools/cts_config/webview_cts_gcs_path.json

 {
+  "_usage":
+  {
+    " The format of this file is as follows.  The excludes and includes lists under test_runs are mutually exclusive, if not specified, all tests in apk will run.":
+    {
+      "<arch>": {
+        "<android SDK Code, such as L, M, N, ...>": {
+          "filename": "<relative path to cts_archive_dir of cts zip>",
+          "_origin":  "<branch@buildid>",
+          "unzip_dir":   "<relative path to work directory where cts should be unzipped to>",
+          "test_runs": [
+          {
+            "apk": "location of the test apk in the cts zip file",
+            "excludes": [
+            {
+              "match": "<class#testcase (wildcard supported) expression of test to skip>",
+              "_bug_id": "<bug reference comment, optional>"
+            }]
+          },
+          {
+            "apk": "location of the test apk in the cts zip file",
+            "includes": [
+            {
+              "match": "<class#testcase (wildcard supported) expression of test to run>"
+            }]
+          }]
+        }
+      }
+    }
+  },
   "arm64": {
     "L": {
-        "filename": "arm64/L/android-cts-arm_64-4607260.zip",
-        "apkdir": "arm64/L/4607260/",
-        "tests": [
-                {"android-cts/repository/testcases/CtsWebkitTestCases.apk": ""},
-                {"android-cts/repository/testcases/CtsWidgetTestCases.apk": "android.widget.cts.RemoteViewsActivityTest#testWebView"}
-        ],
-        "_origin": "aosp-lollipop-mr1-cts-dev@4607260"
+      "filename": "arm64/L/android-cts-arm_64-4607260.zip",
+      "_origin": "aosp-lollipop-mr1-cts-dev@4607260",
+      "unzip_dir": "arm64/L/4607260/",
+      "test_runs": [
+      {
+        "apk": "android-cts/repository/testcases/CtsWebkitTestCases.apk"
+      },
+      {
+        "apk": "android-cts/repository/testcases/CtsWidgetTestCases.apk",
+        "includes": [
+        {
+          "match": "android.widget.cts.RemoteViewsActivityTest#testWebView"
+        }]
+      }]
     },
     "M": {
-        "filename": "arm64/M/android-cts-arm_64-4607285.zip",
-        "apkdir": "arm64/M/4607285/",
-        "tests": [
-                {"android-cts/repository/testcases/CtsWebkitTestCases.apk": ""},
-                {"android-cts/repository/testcases/CtsWidgetTestCases.apk": "android.widget.cts.RemoteViewsActivityTest#testWebView"}
-        ],
-        "_origin": "aosp-marshmallow-cts-dev@4607285"
+      "filename": "arm64/M/android-cts-arm_64-4607285.zip",
+      "_origin": "aosp-marshmallow-cts-dev@4607285",
+      "unzip_dir": "arm64/M/4607285/",
+      "test_runs": [
+      {
+        "apk": "android-cts/repository/testcases/CtsWebkitTestCases.apk"
+      },
+      {
+        "apk": "android-cts/repository/testcases/CtsWidgetTestCases.apk",
+        "includes": [
+        {
+          "match": "android.widget.cts.RemoteViewsActivityTest#testWebView"
+        }]
+      }]
     },
     "N": {
-        "filename": "arm64/N/android-cts-arm_64-4606956.zip",
-        "apkdir": "arm64/N/4606956/",
-        "tests": [
-                {"android-cts/testcases/CtsWebkitTestCases.apk": ""},
-                {"android-cts/testcases/CtsWidgetTestCases.apk": "android.widget.cts.RemoteViewsActivityTest#testWebView"}
-        ],
-        "_origin": "aosp-nougat-cts-release@4606956"
+      "filename": "arm64/N/android-cts-arm_64-4606956.zip",
+      "_origin": "aosp-nougat-cts-release@4606956",
+      "unzip_dir": "arm64/N/4606956/",
+      "test_runs": [
+      {
+        "apk": "android-cts/testcases/CtsWebkitTestCases.apk"
+      },
+      {
+        "apk": "android-cts/testcases/CtsWidgetTestCases.apk",
+        "includes": [
+        {
+          "match": "android.widget.cts.RemoteViewsActivityTest#testWebView"
+        }]
+      }]
     },
     "O": {
-        "filename": "arm64/O/android-cts-arm_64-4666824.zip",
-        "apkdir": "arm64/O/4666824/",
-        "tests": [
-                {"android-cts/testcases/CtsWebkitTestCases.apk": ""},
-                {"android-cts/testcases/CtsWebViewStartupApp.apk": ""}
-        ],
-        "_origin": "aosp-oreo-cts-release@4666824"
+      "filename": "arm64/O/android-cts-arm_64-4666824.zip",
+      "_origin": "aosp-oreo-cts-release@4666824",
+      "unzip_dir": "arm64/O/4666824/",
+      "test_runs": [
+      {
+        "apk": "android-cts/testcases/CtsWebkitTestCases.apk"
+      },
+      {
+        "apk": "android-cts/testcases/CtsWebViewStartupApp.apk"
+      }]
     }
   }
 }

android_webview/tools/run_cts.py

@@ -75,8 +75,24 @@ def GetExpectedFailures():
                               for m in methods])
   return expected_failures
 
+def GetTestRunFilterArg(test_run, skip_expected_failures):
+  skips = []
+  if skip_expected_failures:
+    skips = GetExpectedFailures()
+
+  excludes = test_run.get("excludes", [])
+  includes = test_run.get("includes", [])
+  assert len(excludes) == 0 or len(includes) == 0, \
+         "test_runs error, can't have both includes and excludes: %s" % test_run
+  if len(includes) > 0:
+    return ['-f=' + ':'.join([i["match"] for i in includes])]
+  else:
+    skips.extend([i["match"] for i in excludes])
+    if len(skips) > 0:
+      return ['-f=' + "-" + ':'.join(skips)]
+    return []
 
-def RunCTS(test_runner_args, local_cts_dir, apk, test_filter,
+def RunCTS(test_runner_args, local_cts_dir, test_run,
            skip_expected_failures=True, json_results_file=None):
   """Run tests in apk using test_runner script at _TEST_RUNNER_PATH.
 
@@ -85,20 +101,22 @@ def RunCTS(test_runner_args, local_cts_dir, apk, test_filter,
   is set to False, test results will be stored in
   the json_results_file file if specified
   """
+
+  apk = test_run['apk']
+
   local_test_runner_args = test_runner_args + ['--test-apk',
                                                os.path.join(local_cts_dir, apk)]
 
   # TODO(mikecase): This doesn't work at all with the
   # --gtest-filter test runner option currently. The
   # filter options will just override eachother.
-  if skip_expected_failures:
-    local_test_runner_args += ['-f=-%s' % ':'.join(GetExpectedFailures())]
   # The preferred method is to specify test filters per release in
   # the CTS_GCS path file.  It will override any
   # previous filters, including ones in expected failures
   # file.
-  if test_filter:
-    local_test_runner_args += ['-f=' + test_filter]
+  local_test_runner_args.extend(GetTestRunFilterArg(test_run,
+                                                    skip_expected_failures))
+
   if json_results_file:
     local_test_runner_args += ['--json-results-file=%s' %
                                json_results_file]
@@ -150,7 +168,9 @@ def ExtractCTSZip(args):
 
   cts_zip_path = os.path.join(_CTS_ARCHIVE_DIR, relative_cts_zip_path)
   local_cts_dir = os.path.join(base_cts_dir,
-                               GetCtsInfo(args.arch, args.platform, 'apkdir'))
+                               GetCtsInfo(args.arch, args.platform,
+                                          'unzip_dir')
+                              )
   zf = zipfile.ZipFile(cts_zip_path, 'r')
   zf.extractall(local_cts_dir)
   return (local_cts_dir, base_cts_dir, delete_cts_dir)
@@ -170,26 +190,25 @@ def RunAllCTSTests(args, test_runner_args):
   cts_result = 0
   json_results_file = args.json_results_file
   try:
-    cts_tests_info = GetCtsInfo(args.arch, args.platform, 'tests')
+    cts_test_runs = GetCtsInfo(args.arch, args.platform, 'test_runs')
     cts_results_json = {}
-    for cts_tests_item in cts_tests_info:
-      for relative_apk_path, test_filter in cts_tests_item.iteritems():
-        iteration_cts_result = 0
-        if json_results_file:
-          with tempfile.NamedTemporaryFile() as iteration_json_file:
-            iteration_cts_result = RunCTS(test_runner_args, local_cts_dir,
-                                          relative_apk_path, test_filter,
-                                          args.skip_expected_failures,
-                                          iteration_json_file.name)
-            with open(iteration_json_file.name) as f:
-              additional_results_json = json.load(f)
-              MergeTestResults(cts_results_json, additional_results_json)
-        else:
+    for cts_test_run in cts_test_runs:
+      iteration_cts_result = 0
+      if json_results_file:
+        with tempfile.NamedTemporaryFile() as iteration_json_file:
           iteration_cts_result = RunCTS(test_runner_args, local_cts_dir,
-                                        relative_apk_path, test_filter,
-                                        args.skip_expected_failures)
-        if iteration_cts_result:
-          cts_result = iteration_cts_result
+                                        cts_test_run,
+                                        args.skip_expected_failures,
+                                        iteration_json_file.name)
+          with open(iteration_json_file.name) as f:
+            additional_results_json = json.load(f)
+            MergeTestResults(cts_results_json, additional_results_json)
+      else:
+        iteration_cts_result = RunCTS(test_runner_args, local_cts_dir,
+                                      cts_test_run,
+                                      args.skip_expected_failures)
+      if iteration_cts_result:
+        cts_result = iteration_cts_result
     if json_results_file:
       with open(json_results_file, 'w') as f:
         json.dump(cts_results_json, f, indent=2)