Reland: List Java Instru Test Information From JUnit Runner

This CL allow test information that was once listed by parsing proguard dump or
dexdump to be produced by JUnit runner run all the tests without executing them.
The TestListInstrumentationRunListener is registered in the
BaseChromiumAndroidJUnitRunner and will write all tests to a json file which
then will be pulled to host side by host side runner script

BUG: 640116
Cq-Include-Trybots: master.tryserver.chromium.android:android_cronet_tester
Change-Id: Iea277ea8c7a1d07a2367b1e0a286ce77199935be
Reviewed-on: https://chromium-review.googlesource.com/590694
Commit-Queue: Yoland Yan <yolandyan@chromium.org>
Reviewed-by: Helen Li <xunjieli@chromium.org>
Reviewed-by: Michael Case <mikecase@chromium.org>
Reviewed-by: Tommy Nyquist <nyquist@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Cr-Commit-Position: refs/heads/master@{#491839}

base/BUILD.gn

@@ -2672,6 +2672,7 @@ if (is_android) {
       "test/android/javatests/src/org/chromium/base/test/ScreenshotOnFailureStatement.java",
       "test/android/javatests/src/org/chromium/base/test/SetUpTestRule.java",
       "test/android/javatests/src/org/chromium/base/test/SetUpStatement.java",
+      "test/android/javatests/src/org/chromium/base/test/TestListInstrumentationRunListener.java",
       "test/android/javatests/src/org/chromium/base/test/util/AdvancedMockContext.java",
       "test/android/javatests/src/org/chromium/base/test/util/AnnotationProcessor.java",
       "test/android/javatests/src/org/chromium/base/test/util/CallbackHelper.java",
@@ -2724,6 +2725,7 @@ if (is_android) {
       "android/junit/src/org/chromium/base/process_launcher/ChildConnectionAllocatorTest.java",
       "android/junit/src/org/chromium/base/process_launcher/ChildProcessConnectionTest.java",
       "test/android/junit/src/org/chromium/base/test/SetUpStatementTest.java",
+      "test/android/junit/src/org/chromium/base/test/TestListInstrumentationRunListenerTest.java",
       "test/android/junit/src/org/chromium/base/test/util/DisableIfTest.java",
       "test/android/junit/src/org/chromium/base/test/util/MinAndroidSdkLevelSkipCheckTest.java",
       "test/android/junit/src/org/chromium/base/test/util/RestrictionSkipCheckTest.java",
@@ -2733,6 +2735,7 @@ if (is_android) {
       ":base_java",
       ":base_java_process_launcher_test_support",
       ":base_java_test_support",
+      "//third_party/android_support_test_runner:runner_java",
     ]
     srcjar_deps = [ ":base_build_config_gen" ]
   }

base/test/android/javatests/src/org/chromium/base/test/BaseChromiumAndroidJUnitRunner.java

@@ -4,20 +4,34 @@
 
 package org.chromium.base.test;
 
+import android.app.Activity;
 import android.app.Application;
+import android.app.Instrumentation;
 import android.content.Context;
+import android.os.Bundle;
+import android.support.test.internal.runner.RunnerArgs;
+import android.support.test.internal.runner.TestExecutor;
+import android.support.test.internal.runner.TestRequest;
+import android.support.test.internal.runner.TestRequestBuilder;
 import android.support.test.runner.AndroidJUnitRunner;
 
+import org.chromium.base.Log;
 import org.chromium.base.multidex.ChromiumMultiDexInstaller;
 
 /**
- * A custom AndroidJUnitRunner that supports multidex installer.
+ * A custom AndroidJUnitRunner that supports multidex installer and list out test information.
  *
  * This class is the equivalent of BaseChromiumInstrumentationTestRunner in JUnit3. Please
  * beware that is this not a class runner. It is declared in test apk AndroidManifest.xml
  * <instrumentation>
  */
 public class BaseChromiumAndroidJUnitRunner extends AndroidJUnitRunner {
+    private static final String LIST_ALL_TESTS_FLAG =
+            "org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList";
+    private static final String TAG = "BaseJUnitRunner";
+
+    private Bundle mArguments;
+
     @Override
     public Application newApplication(ClassLoader cl, String className, Context context)
             throws ClassNotFoundException, IllegalAccessException, InstantiationException {
@@ -26,4 +40,54 @@ public class BaseChromiumAndroidJUnitRunner extends AndroidJUnitRunner {
         BaseChromiumRunnerCommon.reorderDexPathElements(cl, getContext(), getTargetContext());
         return super.newApplication(cl, className, context);
     }
+
+    @Override
+    public void onCreate(Bundle arguments) {
+        super.onCreate(arguments);
+        mArguments = arguments;
+    }
+
+    /**
+     * Add TestListInstrumentationRunListener when argument ask the runner to list tests info.
+     *
+     * The running mechanism when argument has "listAllTests" is equivalent to that of
+     * {@link android.support.test.runner.AndroidJUnitRunner#onStart()} except it adds
+     * only TestListInstrumentationRunListener to monitor the tests.
+     */
+    @Override
+    public void onStart() {
+        if (mArguments != null && mArguments.getString(LIST_ALL_TESTS_FLAG) != null) {
+            Log.w(TAG, "Runner will list out tests info in JSON without running tests");
+            listTests(); // Intentionally not calling super.onStart() to avoid additional work.
+        } else {
+            super.onStart();
+        }
+    }
+
+    private void listTests() {
+        Bundle results = new Bundle();
+        try {
+            TestExecutor.Builder executorBuilder = new TestExecutor.Builder(this);
+            executorBuilder.addRunListener(new TestListInstrumentationRunListener(
+                    mArguments.getString(LIST_ALL_TESTS_FLAG)));
+            TestRequest listTestRequest = createListTestRequest(mArguments);
+            results = executorBuilder.build().execute(listTestRequest);
+        } catch (RuntimeException e) {
+            String msg = "Fatal exception when running tests";
+            Log.e(TAG, msg, e);
+            // report the exception to instrumentation out
+            results.putString(Instrumentation.REPORT_KEY_STREAMRESULT,
+                    msg + "\n" + Log.getStackTraceString(e));
+        }
+        finish(Activity.RESULT_OK, results);
+    }
+
+    private TestRequest createListTestRequest(Bundle arguments) {
+        RunnerArgs runnerArgs =
+                new RunnerArgs.Builder().fromManifest(this).fromBundle(arguments).build();
+        TestRequestBuilder builder = new TestRequestBuilder(this, arguments);
+        builder.addApkToScan(getContext().getPackageCodePath());
+        builder.addFromRunnerArgs(runnerArgs);
+        return builder.build();
+    }
 }

base/test/android/javatests/src/org/chromium/base/test/TestListInstrumentationRunListener.java

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.base.test;
+
+import android.support.test.internal.runner.listener.InstrumentationRunListener;
+
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.junit.runner.Description;
+import org.junit.runner.Result;
+
+import org.chromium.base.Log;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * A RunListener that list out all the test information into a json file.
+ */
+public class TestListInstrumentationRunListener extends InstrumentationRunListener {
+    private static final String TAG = "TestListRunListener";
+    private static final Set<String> SKIP_METHODS = new HashSet<>(
+            Arrays.asList(new String[] {"toString", "hashCode", "annotationType", "equals"}));
+
+    private final Map<Class<?>, JSONObject> mTestClassJsonMap = new HashMap<>();
+    private final String mOutputPath;
+
+    public TestListInstrumentationRunListener(String outputPath) {
+        super();
+        mOutputPath = outputPath;
+    }
+
+    /**
+     * Store the test method description to a Map at the beginning of a test run.
+     */
+    @Override
+    public void testStarted(Description desc) throws Exception {
+        if (mTestClassJsonMap.containsKey(desc.getTestClass())) {
+            ((JSONArray) mTestClassJsonMap.get(desc.getTestClass()).get("methods"))
+                .put(getTestMethodJSON(desc));
+        } else {
+            Class<?> testClass = desc.getTestClass();
+            mTestClassJsonMap.put(desc.getTestClass(), new JSONObject()
+                    .put("class", testClass.getName())
+                    .put("superclass", testClass.getSuperclass().getName())
+                    .put("annotations",
+                            getAnnotationJSON(Arrays.asList(testClass.getAnnotations())))
+                    .put("methods", new JSONArray().put(getTestMethodJSON(desc))));
+        }
+    }
+
+    /**
+     * Create a JSONArray with all the test class JSONObjects and save it to listed output path.
+     */
+    @Override
+    public void testRunFinished(Result result) throws IOException {
+        Writer writer = null;
+        File file = new File(mOutputPath);
+        try {
+            writer = new OutputStreamWriter(new FileOutputStream(file), "UTF-8");
+            JSONArray allTestClassesJSON = new JSONArray(mTestClassJsonMap.values());
+            writer.write(allTestClassesJSON.toString());
+        } catch (IOException e) {
+            Log.e(TAG, "failed to write json to file", e);
+            throw e;
+        } finally {
+            if (writer != null) {
+                try {
+                    writer.close();
+                } catch (IOException e) {
+                    // Intentionally ignore IOException when closing writer
+                }
+            }
+        }
+    }
+
+    /**
+     * Return a JSONOject that represent a Description of a method".
+     */
+    static JSONObject getTestMethodJSON(Description desc) throws Exception {
+        return new JSONObject()
+                .put("method", desc.getMethodName())
+                .put("annotations", getAnnotationJSON(desc.getAnnotations()));
+    }
+
+    /**
+     * Create a JSONObject that represent a collection of anntations.
+     *
+     * For example, for the following group of annotations for ExampleClass
+     * <code>
+     * @A
+     * @B(message = "hello", level = 3)
+     * public class ExampleClass() {}
+     * </code>
+     *
+     * This method would return a JSONObject as such:
+     * <code>
+     * {
+     *   "A": {},
+     *   "B": {
+     *     "message": "hello",
+     *     "level": "3"
+     *   }
+     * }
+     * </code>
+     *
+     * The method accomplish this by though through each annotation and reflectively call the
+     * annotation's method to get the element value, with exceptions to methods like "equals()"
+     * or "hashCode".
+     */
+    static JSONObject getAnnotationJSON(Collection<Annotation> annotations)
+            throws Exception {
+        JSONObject annotationsJsons = new JSONObject();
+        for (Annotation a : annotations) {
+            JSONObject elementJsonObject = new JSONObject();
+            for (Method method : a.annotationType().getMethods()) {
+                if (SKIP_METHODS.contains(method.getName())) {
+                    continue;
+                }
+                try {
+                    Object value = method.invoke(a);
+                    if (value == null) {
+                        elementJsonObject.put(method.getName(), null);
+                    } else {
+                        elementJsonObject.put(method.getName(),
+                                value.getClass().isArray()
+                                        ? new JSONArray(Arrays.asList((Object[]) value))
+                                        : value.toString());
+                    }
+                } catch (IllegalArgumentException e) {
+                }
+            }
+            annotationsJsons.put(a.annotationType().getSimpleName(), elementJsonObject);
+        }
+        return annotationsJsons;
+    }
+}

base/test/android/junit/src/org/chromium/base/test/TestListInstrumentationRunListenerTest.java

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.base.test;
+
+import static org.chromium.base.test.TestListInstrumentationRunListener.getAnnotationJSON;
+import static org.chromium.base.test.TestListInstrumentationRunListener.getTestMethodJSON;
+
+import org.json.JSONObject;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.Description;
+import org.junit.runner.RunWith;
+import org.robolectric.annotation.Config;
+
+import org.chromium.base.test.util.CommandLineFlags;
+import org.chromium.testing.local.LocalRobolectricTestRunner;
+
+import java.util.Arrays;
+
+/**
+ * Robolectric test to ensure static methods in TestListInstrumentationRunListener works properly.
+ */
+@RunWith(LocalRobolectricTestRunner.class)
+@Config(manifest = Config.NONE)
+public class TestListInstrumentationRunListenerTest {
+    @CommandLineFlags.Add("hello")
+    private static class ParentClass {
+        public void testA() {}
+
+        @CommandLineFlags.Add("world")
+        public void testB() {}
+    }
+
+    @CommandLineFlags.Remove("hello")
+    private static class ChildClass extends ParentClass {
+    }
+
+    @Test
+    public void testGetTestMethodJSON_testA() throws Throwable {
+        Description desc = Description.createTestDescription(
+                ParentClass.class, "testA",
+                ParentClass.class.getMethod("testA").getAnnotations());
+        JSONObject json = getTestMethodJSON(desc);
+        String expectedJsonString =
+                "{"
+                + "'method': 'testA',"
+                + "'annotations': {}"
+                + "}";
+        expectedJsonString = expectedJsonString
+            .replaceAll("\\s", "")
+            .replaceAll("'", "\"");
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+
+    @Test
+    public void testGetTestMethodJSON_testB() throws Throwable {
+        Description desc = Description.createTestDescription(
+                ParentClass.class, "testB",
+                ParentClass.class.getMethod("testB").getAnnotations());
+        JSONObject json = getTestMethodJSON(desc);
+        String expectedJsonString =
+                "{"
+                + "'method': 'testB',"
+                + "'annotations': {"
+                + "  'Add': {"
+                + "    'value': ['world']"
+                + "    }"
+                + "  }"
+                + "}";
+        expectedJsonString = expectedJsonString
+            .replaceAll("\\s", "")
+            .replaceAll("'", "\"");
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+
+
+    @Test
+    public void testGetTestMethodJSONForInheritedClass() throws Throwable {
+        Description desc = Description.createTestDescription(
+                ChildClass.class, "testB",
+                ChildClass.class.getMethod("testB").getAnnotations());
+        JSONObject json = getTestMethodJSON(desc);
+        String expectedJsonString =
+                "{"
+                + "'method': 'testB',"
+                + "'annotations': {"
+                + "  'Add': {"
+                + "    'value': ['world']"
+                + "    }"
+                + "  }"
+                + "}";
+        expectedJsonString = expectedJsonString
+            .replaceAll("\\s", "")
+            .replaceAll("'", "\"");
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+
+    @Test
+    public void testGetAnnotationJSONForParentClass() throws Throwable {
+        JSONObject json = getAnnotationJSON(Arrays.asList(ParentClass.class.getAnnotations()));
+        String expectedJsonString = "{'Add':{'value':['hello']}}";
+        expectedJsonString = expectedJsonString
+            .replaceAll("\\s", "")
+            .replaceAll("'", "\"");
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+
+    @Test
+    public void testGetAnnotationJSONForChildClass() throws Throwable {
+        JSONObject json = getAnnotationJSON(Arrays.asList(ChildClass.class.getAnnotations()));
+        String expectedJsonString = "{'Add':{'value':['hello']},'Remove':{'value':['hello']}}";
+        expectedJsonString = expectedJsonString
+            .replaceAll("\\s", "")
+            .replaceAll("'", "\"");
+        Assert.assertEquals(expectedJsonString, json.toString());
+    }
+}
+

build/android/pylib/instrumentation/instrumentation_test_instance.py

@@ -38,6 +38,14 @@ _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS = [
     'DisabledTest', 'FlakyTest']
 _VALID_ANNOTATIONS = set(['Manual'] + _DEFAULT_ANNOTATIONS +
                          _EXCLUDE_UNLESS_REQUESTED_ANNOTATIONS)
+
+# These test methods are inherited from android.test base test class and
+# should be permitted for not having size annotation. For more, please check
+# https://developer.android.com/reference/android/test/AndroidTestCase.html
+# https://developer.android.com/reference/android/test/ServiceTestCase.html
+_TEST_WITHOUT_SIZE_ANNOTATIONS = [
+    'testAndroidTestCaseSetupProperly', 'testServiceTestCaseSetUpProperly']
+
 _EXTRA_DRIVER_TEST_LIST = (
     'org.chromium.test.driver.OnDeviceInstrumentationDriver.TestList')
 _EXTRA_DRIVER_TEST_LIST_FILE = (
@@ -69,7 +77,7 @@ class TestListPickleException(test_exception.TestException):
 
 
 # TODO(jbudorick): Make these private class methods of
-# InstrumentationTestInstance once the instrumentation test_runner is
+# InstrumentationTestInstance once the instrumentation junit3_runner_class is
 # deprecated.
 def ParseAmInstrumentRawOutput(raw_output):
   """Parses the output of an |am instrument -r| call.
@@ -165,8 +173,8 @@ def FilterTests(tests, test_filter=None, annotations=None,
 
   Args:
     tests: a list of tests. e.g. [
-           {'annotations": {}, 'class': 'com.example.TestA', 'methods':[]},
-           {'annotations": {}, 'class': 'com.example.TestB', 'methods':[]}]
+           {'annotations": {}, 'class': 'com.example.TestA', 'method':'test1'},
+           {'annotations": {}, 'class': 'com.example.TestB', 'method':'test2'}]
     test_filter: googletest-style filter string.
     annotations: a dict of wanted annotations for test methods.
     exclude_annotations: a dict of annotations to exclude.
@@ -188,6 +196,12 @@ def FilterTests(tests, test_filter=None, annotations=None,
       GetUniqueTestName(t, sep='.')
     ]
 
+    if t['is_junit4']:
+      names += [
+          GetTestNameWithoutParameterPostfix(t, sep='.'),
+          GetTestNameWithoutParameterPostfix(unqualified_class_test, sep='.')
+      ]
+
     pattern_groups = test_filter.split('-')
     if len(pattern_groups) > 1:
       negative_filter = pattern_groups[1]
@@ -230,7 +244,8 @@ def FilterTests(tests, test_filter=None, annotations=None,
       continue
 
     # Enforce that all tests declare their size.
-    if not any(a in _VALID_ANNOTATIONS for a in t['annotations']):
+    if (not any(a in _VALID_ANNOTATIONS for a in t['annotations'])
+        and t['method'] not in _TEST_WITHOUT_SIZE_ANNOTATIONS):
       raise MissingSizeAnnotationError(GetTestName(t))
 
     if (not annotation_filter(t['annotations'])
@@ -242,30 +257,31 @@ def FilterTests(tests, test_filter=None, annotations=None,
   return filtered_tests
 
 
+# TODO(yolandyan): remove this once the tests are converted to junit4
 def GetAllTestsFromJar(test_jar):
   pickle_path = '%s-proguard.pickle' % test_jar
   try:
-    tests = _GetTestsFromPickle(pickle_path, test_jar)
+    tests = GetTestsFromPickle(pickle_path, test_jar)
   except TestListPickleException as e:
     logging.info('Could not get tests from pickle: %s', e)
     logging.info('Getting tests from JAR via proguard.')
     tests = _GetTestsFromProguard(test_jar)
-    _SaveTestsToPickle(pickle_path, test_jar, tests)
+    SaveTestsToPickle(pickle_path, test_jar, tests)
   return tests
 
 
 def GetAllTestsFromApk(test_apk):
   pickle_path = '%s-dexdump.pickle' % test_apk
   try:
-    tests = _GetTestsFromPickle(pickle_path, test_apk)
+    tests = GetTestsFromPickle(pickle_path, test_apk)
   except TestListPickleException as e:
     logging.info('Could not get tests from pickle: %s', e)
     logging.info('Getting tests from dex via dexdump.')
     tests = _GetTestsFromDexdump(test_apk)
-    _SaveTestsToPickle(pickle_path, test_apk, tests)
+    SaveTestsToPickle(pickle_path, test_apk, tests)
   return tests
 
-def _GetTestsFromPickle(pickle_path, jar_path):
+def GetTestsFromPickle(pickle_path, jar_path):
   if not os.path.exists(pickle_path):
     raise TestListPickleException('%s does not exist.' % pickle_path)
   if os.path.getmtime(pickle_path) <= os.path.getmtime(jar_path):
@@ -340,8 +356,7 @@ def _GetTestsFromDexdump(test_apk):
         })
   return tests
 
-
-def _SaveTestsToPickle(pickle_path, jar_path, tests):
+def SaveTestsToPickle(pickle_path, jar_path, tests):
   jar_md5 = md5sum.CalculateHostMd5Sums(jar_path)[jar_path]
   pickle_data = {
     'VERSION': _PICKLE_FORMAT_VERSION,
@@ -383,6 +398,28 @@ def GetTestName(test, sep='#'):
   return '%s%s%s' % (test['class'], sep, test['method'])
 
 
+def GetTestNameWithoutParameterPostfix(
+      test, sep='#', parameterization_sep='__'):
+  """Gets the name of the given JUnit4 test without parameter postfix.
+
+  For most WebView JUnit4 javatests, each test is parameterizatized with
+  "__sandboxed_mode" to run in both non-sandboxed mode and sandboxed mode.
+
+  This function returns the name of the test without parameterization
+  so test filters can match both parameterized and non-parameterized tests.
+
+  Args:
+    test: the instrumentation test dict.
+    sep: the character(s) that should join the class name and the method name.
+    parameterization_sep: the character(s) that seperate method name and method
+                          parameterization postfix.
+  Returns:
+    The test name without parameter postfix as a string.
+  """
+  name = GetTestName(test, sep=sep)
+  return name.split(parameterization_sep)[0]
+
+
 def GetUniqueTestName(test, sep='#'):
   """Gets the unique name of the given test.
 
@@ -415,8 +452,8 @@ class InstrumentationTestInstance(test_instance.TestInstance):
     self._test_apk_incremental_install_script = None
     self._test_jar = None
     self._test_package = None
-    self._test_runner = None
-    self._test_runner_junit4 = None
+    self._junit3_runner_class = None
+    self._junit4_runner_class = None
     self._test_support_apk = None
     self._initializeApkAttributes(args, error_func)
 
@@ -524,23 +561,25 @@ class InstrumentationTestInstance(test_instance.TestInstance):
 
     self._test_package = self._test_apk.GetPackageName()
     all_instrumentations = self._test_apk.GetAllInstrumentations()
-    test_runners = [
+    all_junit3_runner_classes = [
         x for x in all_instrumentations if ('true' not in x.get(
             'chromium-junit4', ''))]
-    test_runners_junit4 = [
+    all_junit4_test_runner_classes = [
         x for x in all_instrumentations if ('true' in x.get(
             'chromium-junit4', ''))]
 
-    if len(test_runners) > 1:
+    if len(all_junit3_runner_classes) > 1:
       logging.warning('This test apk has more than one JUnit3 instrumentation')
-    if len(test_runners_junit4) > 1:
+    if len(all_junit4_test_runner_classes) > 1:
       logging.warning('This test apk has more than one JUnit4 instrumentation')
 
-    self._test_runner = (
-      test_runners[0]['android:name'] if test_runners else
-      self.test_apk.GetInstrumentationName())
-    self._test_runner_junit4 = (
-      test_runners_junit4[0]['android:name'] if test_runners_junit4 else None)
+    self._junit3_runner_class = (
+      all_junit3_runner_classes[0]['android:name']
+      if all_junit3_runner_classes else self.test_apk.GetInstrumentationName())
+
+    self._junit4_runner_class = (
+      all_junit4_test_runner_classes[0]['android:name']
+      if all_junit4_test_runner_classes else None)
 
     self._package_info = None
     if self._apk_under_test:
@@ -696,6 +735,14 @@ class InstrumentationTestInstance(test_instance.TestInstance):
   def gs_results_bucket(self):
     return self._gs_results_bucket
 
+  @property
+  def junit3_runner_class(self):
+    return self._junit3_runner_class
+
+  @property
+  def junit4_runner_class(self):
+    return self._junit4_runner_class
+
   @property
   def should_save_logcat(self):
     return self._should_save_logcat
@@ -748,14 +795,6 @@ class InstrumentationTestInstance(test_instance.TestInstance):
   def test_package(self):
     return self._test_package
 
-  @property
-  def test_runner(self):
-    return self._test_runner
-
-  @property
-  def test_runner_junit4(self):
-    return self._test_runner_junit4
-
   @property
   def timeout_scale(self):
     return self._timeout_scale
@@ -782,11 +821,15 @@ class InstrumentationTestInstance(test_instance.TestInstance):
 
   def GetTests(self):
     if self.test_jar:
-      tests = GetAllTestsFromJar(self.test_jar)
+      raw_tests = GetAllTestsFromJar(self.test_jar)
     else:
-      tests = GetAllTestsFromApk(self.test_apk.path)
-    inflated_tests = self._ParameterizeTestsWithFlags(self._InflateTests(tests))
-    if self._test_runner_junit4 is None and any(
+      raw_tests = GetAllTestsFromApk(self.test_apk.path)
+    return self.ProcessRawTests(raw_tests)
+
+  def ProcessRawTests(self, raw_tests):
+    inflated_tests = self._ParameterizeTestsWithFlags(
+        self._InflateTests(raw_tests))
+    if self._junit4_runner_class is None and any(
         t['is_junit4'] for t in inflated_tests):
       raise MissingJUnit4RunnerException()
     filtered_tests = FilterTests(
@@ -833,7 +876,7 @@ class InstrumentationTestInstance(test_instance.TestInstance):
       self, test_list=None, test_list_file_path=None):
     env = {
       _EXTRA_DRIVER_TARGET_PACKAGE: self.test_package,
-      _EXTRA_DRIVER_TARGET_CLASS: self.test_runner,
+      _EXTRA_DRIVER_TARGET_CLASS: self.junit3_runner_class,
       _EXTRA_TIMEOUT_SCALE: self._timeout_scale,
     }
 

build/android/pylib/local/device/local_device_instrumentation_test_run.py

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import contextlib
+import json
 import logging
 import os
 import posixpath
@@ -64,6 +65,12 @@ EXTRA_SCREENSHOT_FILE = (
 EXTRA_UI_CAPTURE_DIR = (
     'org.chromium.base.test.util.Screenshooter.ScreenshotDir')
 
+_EXTRA_TEST_LIST = (
+    'org.chromium.base.test.BaseChromiumAndroidJUnitRunner.TestList')
+
+_TEST_LIST_JUNIT4_RUNNERS = [
+    'org.chromium.base.test.BaseChromiumAndroidJUnitRunner']
+
 UI_CAPTURE_DIRS = ['chromium_tests_root', 'UiCapture']
 
 FEATURE_ANNOTATION = 'Feature'
@@ -87,8 +94,8 @@ def _LogTestEndpoints(device, test_name):
         ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
         check_return=True)
 
-# TODO(jbudorick): Make this private once the instrumentation test_runner is
-# deprecated.
+# TODO(jbudorick): Make this private once the instrumentation test_runner
+# is deprecated.
 def DidPackageCrashOnDevice(package_name, device):
   # Dismiss any error dialogs. Limit the number in case we have an error
   # loop or we are failing to dismiss.
@@ -305,7 +312,12 @@ class LocalDeviceInstrumentationTestRun(
 
   #override
   def _GetTests(self):
-    tests = self._test_instance.GetTests()
+    tests = None
+    if self._test_instance.junit4_runner_class in _TEST_LIST_JUNIT4_RUNNERS:
+      raw_tests = self._GetTestsFromRunner()
+      tests = self._test_instance.ProcessRawTests(raw_tests)
+    else:
+      tests = self._test_instance.GetTests()
     tests = self._ApplyExternalSharding(
         tests, self._test_instance.external_shard_index,
         self._test_instance.total_external_shards)
@@ -370,10 +382,11 @@ class LocalDeviceInstrumentationTestRun(
       if test['is_junit4']:
         target = '%s/%s' % (
             self._test_instance.test_package,
-            self._test_instance.test_runner_junit4)
+            self._test_instance.junit4_runner_class)
       else:
         target = '%s/%s' % (
-            self._test_instance.test_package, self._test_instance.test_runner)
+            self._test_instance.test_package,
+            self._test_instance.junit3_runner_class)
       extras['class'] = test_name
       if 'flags' in test and test['flags']:
         flags_to_add.extend(test['flags'])
@@ -551,6 +564,49 @@ class LocalDeviceInstrumentationTestRun(
       post_test_step_thread_group.JoinAll()
     return results, None
 
+  def _GetTestsFromRunner(self):
+    test_apk_path = self._test_instance.test_apk.path
+    pickle_path = '%s-runner.pickle' % test_apk_path
+    try:
+      return instrumentation_test_instance.GetTestsFromPickle(
+          pickle_path, test_apk_path)
+    except instrumentation_test_instance.TestListPickleException as e:
+      logging.info('Could not get tests from pickle: %s', e)
+    logging.info('Getting tests by having %s list them.',
+                 self._test_instance.junit4_runner_class)
+    def list_tests(dev):
+      with device_temp_file.DeviceTempFile(
+          dev.adb, suffix='.json',
+          dir=dev.GetExternalStoragePath()) as dev_test_list_json:
+        junit4_runner_class = self._test_instance.junit4_runner_class
+        test_package = self._test_instance.test_package
+        extras = {}
+        extras['log'] = 'true'
+        extras['package'] = '.'.join(test_package.split('.')[:2])
+        extras[_EXTRA_TEST_LIST] = dev_test_list_json.name
+        target = '%s/%s' % (test_package, junit4_runner_class)
+        dev.StartInstrumentation(target, extras=extras)
+        with tempfile_ext.NamedTemporaryDirectory() as host_dir:
+          host_file = os.path.join(host_dir, 'list_tests.json')
+          dev.PullFile(dev_test_list_json.name, host_file)
+          with open(host_file, 'r') as host_file:
+              return json.load(host_file)
+
+    raw_test_lists = self._env.parallel_devices.pMap(list_tests).pGet(None)
+
+    # If all devices failed to list tests, raise an exception.
+    # Check that tl is not None and is not empty.
+    if all(not tl for tl in raw_test_lists):
+      raise device_errors.CommandFailedError(
+          'Failed to list tests on any device')
+
+    # Get the first viable list of raw tests
+    raw_tests = [tl for tl in raw_test_lists if tl][0]
+
+    instrumentation_test_instance.SaveTestsToPickle(
+        pickle_path, test_apk_path, raw_tests)
+    return raw_tests
+
   def _SaveScreenshot(self, device, screenshot_host_dir, screenshot_device_file,
                       test_name, results):
     if screenshot_host_dir:

components/cronet/android/BUILD.gn

@@ -790,6 +790,8 @@ instrumentation_test_apk(
   apk_under_test = ":cronet_smoketests_platform_only_apk"
   android_manifest = "test/javatests/AndroidManifest.xml"
   deps = [
+    "//base:base_java_test_support",
+    "//third_party/android_support_test_runner:runner_java",
     ":cronet_smoketests_platform_only_java",
   ]
 

components/cronet/android/cronet_impl_platform_proguard.cfg

 # Proguard config for apps that depend on cronet_impl_platform_java.jar.
 
+# https://android.googlesource.com/platform/sdk/+/marshmallow-mr1-release/files/proguard-android.txt#54
+-dontwarn android.support.**
+
 # This constructor is called using the reflection from Cronet API (cronet_api.jar).
 -keep class org.chromium.net.impl.JavaCronetProvider {
     public <init>(android.content.Context);
-}
+}
\ No newline at end of file
+
+# Generated for chrome apk and not included into cronet.
+-dontwarn org.chromium.base.BuildConfig
+-dontwarn org.chromium.base.library_loader.NativeLibraries
+-dontwarn org.chromium.base.multidex.ChromiumMultiDexInstaller

components/cronet/android/test/proguard.cfg

@@ -18,5 +18,6 @@
 # class/merging/horizontal proguard optimization is enabled.
 # NOTE: make sure that only test classes are added to this list.
 -keep class org.chromium.base.test.util.**
+
 -keep class org.chromium.net.TestFilesInstaller
 -keep class org.chromium.net.MetricsTestUtil

testing/android/proguard_for_test.flags

@@ -30,6 +30,15 @@
     public static ** valueOf(java.lang.String);
 }
 
+# Keey any annotation used by tests for instrumentation runner to list out 
+# test annotation information
+-keep @interface android.support.test.**
+-keep @interface org.chromium.base.test.**
+-keep @interface org.junit.**
+-keep @interface android.test.**
+
+-keepattributes *Annotation*
+
 # We have some "library class WebView depends on program class SslCertificate"
 # warnings, and they don't affect us.
 -dontwarn android.webkit.WebView*