android: switch test runner to standalone Avd management module. (RELAND)

This is a reland of crrev.com/c/1797004

Bug: 922145
Change-Id: Ic388afd4cd4303c25057ca77b38cea39d965d3fb
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1827248Reviewed-by: Ben Pastene <bpastene@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: John Budorick <jbudorick@chromium.org>
Cr-Commit-Position: refs/heads/master@{#702055}

.gitignore

@@ -32,10 +32,12 @@
 !Android.mk
 .*.sw?
 .DS_Store
+.android
 .cipd
 .classpath
 .code-coverage
 .cproject
+.emulator_sdk
 .gdb_history
 .gdbinit
 .landmines

build/android/BUILD.gn

@@ -93,11 +93,15 @@ python_library("test_runner_py") {
     data += [
       "//third_party/android_build_tools/bundletool/bundletool-all-0.10.3.jar",
     ]
-    data_deps +=
-        [ "//third_party/android_platform/development/scripts:stack_py" ]
-  }
-  if (is_asan) {
-    data_deps += [ "//tools/android/asan/third_party:asan_device_setup" ]
+    data_deps += [
+      "//third_party/android_platform/development/scripts:stack_py",
+      "//tools/android/avd:avd_py",
+    ]
+    if (is_asan) {
+      data_deps += [ "//tools/android/asan/third_party:asan_device_setup" ]
+    }
+  } else {
+    pydeps_sources_assignment_filters = [ "../../tools/*" ]
   }
 
   # Proguard is needed only when using apks (rather than native executables).

build/android/pylib/base/environment_factory.py

@@ -4,14 +4,24 @@
 
 from pylib import constants
 from pylib.local.device import local_device_environment
-from pylib.local.emulator import local_emulator_environment
 from pylib.local.machine import local_machine_environment
 
+try:
+  # local_emulator_environment depends on //tools.
+  # If a client pulls in the //build subtree but not the //tools
+  # one, fail at emulator environment creation time.
+  from pylib.local.emulator import local_emulator_environment
+except ImportError:
+  local_emulator_environment = None
+
+
 def CreateEnvironment(args, output_manager, error_func):
 
   if args.environment == 'local':
     if args.command not in constants.LOCAL_MACHINE_TESTS:
-      if args.avd_name:
+      if args.avd_config:
+        if not local_emulator_environment:
+          error_func('emulator environment requested but not available.')
         return local_emulator_environment.LocalEmulatorEnvironment(
             args, output_manager, error_func)
       return local_device_environment.LocalDeviceEnvironment(

build/android/pylib/local/emulator/local_emulator_environment.py

@@ -2,120 +2,36 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import contextlib
-import logging
 import os
-import socket
-import stat
 
-from py_utils import tempfile_ext
-
-from devil.android.sdk import adb_wrapper
-from devil.utils import cmd_helper
-from devil.utils import timeout_retry
-
-from pylib import constants
+from pylib.constants import host_paths
 from pylib.local.device import local_device_environment
 
+AVD_DIR_PATH = os.path.join(host_paths.DIR_SOURCE_ROOT, 'tools', 'android',
+                            'avd')
+with host_paths.SysPath(AVD_DIR_PATH):
+  import avd  # pylint: disable=import-error
+
 
 class LocalEmulatorEnvironment(local_device_environment.LocalDeviceEnvironment):
 
   def __init__(self, args, output_manager, error_func):
     super(LocalEmulatorEnvironment, self).__init__(args, output_manager,
                                                    error_func)
-    self._avd_name = args.avd_name
-    self._emulator_home = (args.emulator_home
-                           or os.path.expanduser(os.path.join('~', '.android')))
-
-    root_ini = os.path.join(self._emulator_home, 'avd',
-                            '%s.ini' % self._avd_name)
-    if not os.path.exists(root_ini):
-      error_func('Unable to find configuration for AVD %s at %s' %
-                 (self._avd_name, root_ini))
-
-    self._emulator_path = os.path.join(constants.ANDROID_SDK_ROOT, 'emulator',
-                                       'emulator')
-    if not os.path.exists(self._emulator_path):
-      error_func('%s does not exist.' % self._emulator_path)
-
-    self._emulator_proc = None
-    self._emulator_serial = None
+    self._avd_config = avd.AvdConfig(args.avd_config)
+    self._emulator_instance = None
 
   #override
   def SetUp(self):
-    # Emulator start-up looks for the adb daemon. Make sure it's running.
-    adb_wrapper.AdbWrapper.StartServer()
-
-    # Emulator start-up tries to check for the SDK root by looking for
-    # platforms/ and platform-tools/. Ensure they exist.
-    # See http://bit.ly/2YAkyFE for context.
-    required_dirs = [
-        os.path.join(constants.ANDROID_SDK_ROOT, 'platforms'),
-        os.path.join(constants.ANDROID_SDK_ROOT, 'platform-tools'),
-    ]
-    for d in required_dirs:
-      if not os.path.exists(d):
-        os.makedirs(d)
-
-    # The emulator requires that some files are writable.
-    for dirname, _, filenames in os.walk(self._emulator_home):
-      for f in filenames:
-        path = os.path.join(dirname, f)
-        if (os.lstat(path).st_mode &
-            (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) == stat.S_IRUSR):
-          os.chmod(path, stat.S_IRUSR | stat.S_IWUSR)
-
-    self._emulator_proc, self._emulator_serial = self._StartInstance()
-
-    logging.info('Emulator serial: %s', self._emulator_serial)
-    self._device_serials = [self._emulator_serial]
+    self._avd_config.Install()
+    self._emulator_instance = self._avd_config.StartInstance()
+    self._device_serials = [self._emulator_instance.serial]
     super(LocalEmulatorEnvironment, self).SetUp()
 
-  def _StartInstance(self):
-    """Starts an AVD instance.
-
-    Returns:
-      A (Popen, str) 2-tuple that includes the process and serial.
-    """
-    # Start up the AVD.
-    with tempfile_ext.TemporaryFileName() as socket_path, (contextlib.closing(
-        socket.socket(socket.AF_UNIX))) as sock:
-      sock.bind(socket_path)
-      emulator_cmd = [
-          self._emulator_path,
-          '-avd',
-          self._avd_name,
-          '-report-console',
-          'unix:%s' % socket_path,
-          '-read-only',
-          '-no-window',
-      ]
-      emulator_env = {}
-      if self._emulator_home:
-        emulator_env['ANDROID_EMULATOR_HOME'] = self._emulator_home
-      sock.listen(1)
-      emulator_proc = cmd_helper.Popen(emulator_cmd, env=emulator_env)
-
-      def listen_for_serial(s):
-        logging.info('Waiting for connection from emulator.')
-        with contextlib.closing(s.accept()[0]) as conn:
-          val = conn.recv(1024)
-          return 'emulator-%d' % int(val)
-
-      try:
-        emulator_serial = timeout_retry.Run(
-            listen_for_serial, timeout=30, retries=0, args=[sock])
-      except Exception:
-        emulator_proc.terminate()
-        raise
-
-      return (emulator_proc, emulator_serial)
-
   #override
   def TearDown(self):
     try:
       super(LocalEmulatorEnvironment, self).TearDown()
     finally:
-      if self._emulator_proc:
-        self._emulator_proc.terminate()
-        self._emulator_proc.wait()
+      if self._emulator_instance:
+        self._emulator_instance.Stop()

build/android/test_runner.py

@@ -310,13 +310,11 @@ def AddEmulatorOptions(parser):
   parser = parser.add_argument_group('emulator arguments')
 
   parser.add_argument(
-      '--avd-name',
-      help='Run and manage the lifetime of an AVD with the given name.')
-  parser.add_argument(
-      '--emulator-home',
+      '--avd-config',
       type=os.path.realpath,
-      help='Emulator home directory '
-      '(see ANDROID_EMULATOR_HOME: http://bit.ly/2K32oEy)')
+      help='Path to the avd config textpb. '
+      '(See //tools/android/avd/proto/ for message definition'
+      ' and existing textpb files.)')
 
 
 def AddGTestOptions(parser):

build/android/test_runner.pydeps

@@ -118,6 +118,9 @@
 ../../third_party/markupsafe/__init__.py
 ../../third_party/markupsafe/_compat.py
 ../../third_party/markupsafe/_native.py
+../../tools/android/avd/avd.py
+../../tools/android/avd/proto/__init__.py
+../../tools/android/avd/proto/avd_pb2.py
 ../../tools/swarming_client/libs/__init__.py
 ../../tools/swarming_client/libs/logdog/__init__.py
 ../../tools/swarming_client/libs/logdog/bootstrap.py

build/config/python.gni

@@ -22,6 +22,9 @@
 #
 # Variables
 #   pydeps_file: Path to .pydeps file to read sources from (optional).
+#   pydeps_sources_assignment_filters: Additional sources_assignment_filters to
+#     use when reading the contents of the pydeps file. Unused if pydeps_file
+#     isn't set. (optional)
 #   data: Additional files to include in data. E.g. non-.py files needed by the
 #         library, or .py files that are conditionally / lazily imported.
 #
@@ -44,7 +47,12 @@ template("python_library") {
       _py_files = read_file(invoker.pydeps_file, "list lines")
 
       # Filter out comments.
-      set_sources_assignment_filter([ "#*" ])
+      pydeps_sources_assignment_filters = [ "#*" ]
+      if (defined(invoker.pydeps_sources_assignment_filters)) {
+        pydeps_sources_assignment_filters +=
+            invoker.pydeps_sources_assignment_filters
+      }
+      set_sources_assignment_filter(pydeps_sources_assignment_filters)
       sources = _py_files
 
       # Dependencies are listed relative to the pydeps file directory, but data

testing/buildbot/mixins.pyl

@@ -548,32 +548,24 @@
   'pie-x86-emulator': {
     '$mixin_append': {
       'args': [
-        '--avd-name=20190716T160000Z_android_28_google_apis_x86',
-        '--emulator-home=../../.android',
+        '--avd-config=../../tools/android/avd/proto/generic_android28.textpb',
       ],
     },
     'swarming': {
-      'cipd_packages': [
-        {
-          'cipd_package': 'chromium/third_party/android_sdk/public/avds/android-28/google_apis/x86',
-          'location': '.android',
-          'revision': 'J5MFTg7OhekLKsOX7bmkVTSCDzohydCvXrn4sJzy0xsC',
-        },
-        {
-          'cipd_package': 'chromium/third_party/android_sdk/public/emulator',
-          'location': 'third_party/android_sdk/public',
-          'revision': 'xhyuoquVvBTcJelgRjMKZeoBVSQRjB7pLVJPt5C9saIC',
-        },
-        {
-          'cipd_package': 'chromium/third_party/android_sdk/public/system-images/android-28/google_apis/x86',
-          'location': 'third_party/android_sdk/public',
-          'revision': 'LDa0XkTjgGYx7Amzg5qjIRgCfc4F_pq7rKMJVdACYx8C',
-        },
-      ],
       'dimensions': {
         'device_os': None,
         'device_type': None,
       },
+      'named_caches': [
+        {
+          'name': 'avd_generic_android28',
+          'path': '.android',
+        },
+        {
+          'name': 'system_images_android_28_google_apis_x86',
+          'path': '.emulator_sdk',
+        },
+      ]
     },
   },
   'pie_fleet': {