[Fuchsia] Allow downloading of astro images via hooks.

Create --internal-images flag. When used, the update_sdk script will download astro images to //third_party/fuchsia-internal-images.
Include internal images in test target if target_cpu is arm64 and x64 host tools are needed.

Bug: 1134108
Change-Id: If323130add41543e7a7a7f50acbfea80c8ed790b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2446832
Commit-Queue: Chong Gu <chonggu@google.com>
Reviewed-by: Kevin Marshall <kmarshall@chromium.org>
Auto-Submit: Chong Gu <chonggu@google.com>
Cr-Commit-Position: refs/heads/master@{#814047}

DEPS

@@ -4931,12 +4931,22 @@ hooks = [
   },
 
   {
-    'name': 'fuchsia_sdk',
+    'name': 'Download Fuchsia SDK',
     'pattern': '.',
     'condition': 'checkout_fuchsia',
     'action': [
       'python',
       'src/build/fuchsia/update_sdk.py',
+    ],
+  },
+
+  {
+    'name': 'Download Fuchsia system images',
+    'pattern': '.',
+    'condition': 'checkout_fuchsia',
+    'action': [
+      'python',
+      'src/build/fuchsia/update_images.py',
       '--boot-images={checkout_fuchsia_boot_images}',
     ],
   },

build/config/fuchsia/generate_runner_scripts.gni

@@ -20,6 +20,10 @@ declare_args() {
   # Architecture of the host tools included in Fuchsia test targets.
   # Defaults to target_cpu.
   fuchsia_override_host_tool_arch_for_isolated_testing = target_cpu
+
+  # A list that contains additional Fuchsia boot images to include in the test
+  # targets.
+  fuchsia_additional_boot_images = []
 }
 
 # Generates a script which deploys and optionally executes a package on a
@@ -114,6 +118,7 @@ template("fuchsia_package_runner") {
       "//third_party/fuchsia-sdk/sdk/tools/${fuchsia_override_host_tool_arch_for_isolated_testing}/symbolize",
       "//third_party/fuchsia-sdk/sdk/tools/${fuchsia_override_host_tool_arch_for_isolated_testing}/zbi",
     ]
+
     if (target_cpu == "arm64") {
       data += [ "${qemu_arm64_root}/" ]
     } else {
@@ -123,6 +128,10 @@ template("fuchsia_package_runner") {
       ]
     }
 
+    foreach(fuchsia_additional_boot_image, fuchsia_additional_boot_images) {
+      data += [ "${fuchsia_additional_boot_images}/" ]
+    }
+
     executable_args = []
 
     package_paths = [ rebase_path(_package_path, root_build_dir) ]

build/fuchsia/update_images.py

+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Updates the Fuchsia SDK to the given revision. Should be used in a 'hooks_os'
+entry so that it only runs when .gclient's target_os includes 'fuchsia'."""
+
+import argparse
+import itertools
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tarfile
+
+from common import GetHostOsFromPlatform, GetHostArchFromPlatform, \
+                   DIR_SOURCE_ROOT, IMAGES_ROOT
+from update_sdk import DownloadAndUnpackFromCloudStorage, GetSdkHash, \
+                       MakeCleanDirectory, SDK_SIGNATURE_FILE
+
+
+def GetSdkSignature(sdk_hash, boot_images):
+  return 'gn:{sdk_hash}:{boot_images}:'.format(sdk_hash=sdk_hash,
+                                               boot_images=boot_images)
+
+
+def GetAllImages(boot_image_names):
+  if not boot_image_names:
+    return
+
+  all_device_types = ['generic', 'qemu']
+  all_archs = ['x64', 'arm64']
+
+  images_to_download = set()
+
+  for boot_image in boot_image_names.split(','):
+    components = boot_image.split('.')
+    if len(components) != 2:
+      continue
+
+    device_type, arch = components
+    device_images = all_device_types if device_type == '*' else [device_type]
+    arch_images = all_archs if arch == '*' else [arch]
+    images_to_download.update(itertools.product(device_images, arch_images))
+  return images_to_download
+
+
+def DownloadSdkBootImages(bucket, sdk_hash, boot_image_names, image_output_dir):
+  images_to_download = GetAllImages(boot_image_names)
+  for image_to_download in images_to_download:
+    device_type = image_to_download[0]
+    arch = image_to_download[1]
+    image_output_dir = os.path.join(image_output_dir, arch, device_type)
+    if os.path.exists(image_output_dir):
+      continue
+
+    logging.info('Downloading Fuchsia boot images for %s.%s...' %
+                 (device_type, arch))
+    if bucket == 'fuchsia':
+      images_tarball_url = 'gs://{bucket}/development/{sdk_hash}/images/'\
+          '{device_type}-{arch}.tgz'.format(
+              bucket=bucket, sdk_hash=sdk_hash,
+              device_type=device_type, arch=arch)
+    else:
+      images_tarball_url = 'gs://{bucket}/development/{sdk_hash}/images/'\
+          '{device_type}.{arch}.tgz'.format(
+              bucket=bucket, sdk_hash=sdk_hash,
+              device_type=device_type, arch=arch)
+    DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir)
+
+
+def GetNewSignature(sdk_hash, boot_images):
+  return GetSdkSignature(sdk_hash, boot_images)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--verbose',
+                      '-v',
+                      action='store_true',
+                      help='Enable debug-level logging.')
+  parser.add_argument(
+      '--boot-images',
+      type=str,
+      required=True,
+      nargs='?',
+      help='List of boot images to download, represented as a comma separated '
+      'list. Wildcards are allowed. ')
+  parser.add_argument(
+      '--bucket',
+      type=str,
+      default='fuchsia',
+      help='The cloud bucket in which the Fuchsia images are stored. Optional')
+  parser.add_argument(
+      '--image-root-dir',
+      default=IMAGES_ROOT,
+      help='Specify the root directory of the downloaded images. Optional')
+  args = parser.parse_args()
+
+  logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
+
+  # If no boot images need to be downloaded, exit.
+  if not args.boot_images:
+    return 0
+
+  # Check whether there's SDK support for this platform.
+  GetHostOsFromPlatform()
+
+  sdk_hash = GetSdkHash(args.bucket)
+  if not sdk_hash:
+    return 1
+
+  signature_filename = os.path.join(args.image_root_dir, SDK_SIGNATURE_FILE)
+  current_signature = (open(signature_filename, 'r').read().strip()
+                       if os.path.exists(signature_filename) else '')
+  new_signature = GetNewSignature(sdk_hash, args.boot_images)
+  if current_signature != new_signature:
+    logging.info('Downloading Fuchsia images %s...' % sdk_hash)
+    MakeCleanDirectory(args.image_root_dir)
+
+    try:
+      DownloadSdkBootImages(args.bucket, sdk_hash, args.boot_images,
+                            args.image_root_dir)
+    except subprocess.CalledProcessError as e:
+      logging.error(("command '%s' failed with status %d.%s"), " ".join(e.cmd),
+                    e.returncode, " Details: " + e.output if e.output else "")
+
+    with open(signature_filename, 'w') as f:
+      f.write(new_signature)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())

build/fuchsia/update_sdk.py

@@ -7,7 +7,6 @@
 entry so that it only runs when .gclient's target_os includes 'fuchsia'."""
 
 import argparse
-import itertools
 import logging
 import os
 import re
@@ -17,14 +16,12 @@ import sys
 import tarfile
 
 from common import GetHostOsFromPlatform, GetHostArchFromPlatform, \
-                   DIR_SOURCE_ROOT, SDK_ROOT, IMAGES_ROOT
+                   DIR_SOURCE_ROOT, SDK_ROOT
 
 sys.path.append(os.path.join(DIR_SOURCE_ROOT, 'build'))
 import find_depot_tools
 
 SDK_SIGNATURE_FILE = '.hash'
-
-EXTRA_SDK_HASH_PREFIX = ''
 SDK_TARBALL_PATH_TEMPLATE = (
     'gs://{bucket}/development/{sdk_hash}/sdk/{platform}-amd64/gn.tar.gz')
 
@@ -78,16 +75,6 @@ def GetSdkTarballPath(bucket, sdk_hash):
       bucket=bucket, sdk_hash=sdk_hash, platform=GetHostOsFromPlatform())
 
 
-def GetSdkSignature(sdk_hash, boot_images):
-  return 'gn:{sdk_hash}:{boot_images}:'.format(
-      sdk_hash=sdk_hash, boot_images=boot_images)
-
-
-def EnsureDirExists(path):
-  if not os.path.exists(path):
-    os.makedirs(path)
-
-
 # Updates the modification timestamps of |path| and its contents to the
 # current time.
 def UpdateTimestampsRecursive():
@@ -120,37 +107,10 @@ def DownloadAndUnpackFromCloudStorage(url, output_dir):
                                         task.stderr.read())
 
 
-def DownloadSdkBootImages(bucket, sdk_hash, boot_image_names):
-  if not boot_image_names:
-    return
-
-  all_device_types = ['generic', 'qemu']
-  all_archs = ['x64', 'arm64']
-
-  images_to_download = set()
-  for boot_image in boot_image_names.split(','):
-    components = boot_image.split('.')
-    if len(components) != 2:
-      continue
-
-    device_type, arch = components
-    device_images = all_device_types if device_type=='*' else [device_type]
-    arch_images = all_archs if arch=='*' else [arch]
-    images_to_download.update(itertools.product(device_images, arch_images))
-
-  for image_to_download in images_to_download:
-    device_type = image_to_download[0]
-    arch = image_to_download[1]
-    image_output_dir = os.path.join(IMAGES_ROOT, arch, device_type)
-    if os.path.exists(image_output_dir):
-      continue
-
-    logging.info(
-        'Downloading Fuchsia boot images for %s.%s...' % (device_type, arch))
-    images_tarball_url = 'gs://{bucket}/development/{sdk_hash}/images/'\
-        '{device_type}-{arch}.tgz'.format(
-            bucket=bucket, sdk_hash=sdk_hash, device_type=device_type, arch=arch)
-    DownloadAndUnpackFromCloudStorage(images_tarball_url, image_output_dir)
+def MakeCleanDirectory(directory_name):
+  if (os.path.exists(directory_name)):
+    shutil.rmtree(directory_name)
+  os.mkdir(directory_name)
 
 
 def main():
@@ -158,11 +118,6 @@ def main():
   parser.add_argument('--verbose', '-v',
     action='store_true',
     help='Enable debug-level logging.')
-  parser.add_argument('--boot-images',
-    type=str, nargs='?',
-    help='List of boot images to download, represented as a comma separated '
-         'list. Wildcards are allowed. '
-         'If omitted, no boot images will be downloaded.')
   args = parser.parse_args()
 
   logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO)
@@ -174,6 +129,7 @@ def main():
     return 0
 
   bucket = GetCloudStorageBucket()
+
   sdk_hash = GetSdkHash(bucket)
   if not sdk_hash:
     return 1
@@ -181,33 +137,15 @@ def main():
   signature_filename = os.path.join(SDK_ROOT, SDK_SIGNATURE_FILE)
   current_signature = (open(signature_filename, 'r').read().strip()
                        if os.path.exists(signature_filename) else '')
-  if current_signature != GetSdkSignature(sdk_hash, args.boot_images):
+  if current_signature != sdk_hash:
     logging.info('Downloading GN SDK %s...' % sdk_hash)
 
-    if os.path.isdir(SDK_ROOT):
-      shutil.rmtree(SDK_ROOT)
-
-    EnsureDirExists(SDK_ROOT)
-    DownloadAndUnpackFromCloudStorage(
-        GetSdkTarballPath(bucket, sdk_hash), SDK_ROOT)
-
-    # Clean out the boot images directory.
-    if (os.path.exists(IMAGES_ROOT)):
-      shutil.rmtree(IMAGES_ROOT)
-      os.mkdir(IMAGES_ROOT)
-
-    try:
-      # Ensure that the boot images are downloaded for this SDK.
-      # If the developer opted into downloading hardware boot images in their
-      # .gclient file, then only the hardware boot images will be downloaded.
-      DownloadSdkBootImages(bucket, sdk_hash, args.boot_images)
-    except subprocess.CalledProcessError as e:
-      logging.error(("command '%s' failed with status %d.%s"), " ".join(e.cmd),
-                    e.returncode, " Details: " + e.output if e.output else "")
-      return 1
+    MakeCleanDirectory(SDK_ROOT)
+    DownloadAndUnpackFromCloudStorage(GetSdkTarballPath(bucket, sdk_hash),
+                                      SDK_ROOT)
 
   with open(signature_filename, 'w') as f:
-    f.write(GetSdkSignature(sdk_hash, args.boot_images))
+    f.write(sdk_hash)
 
   UpdateTimestampsRecursive()
 

third_party/.gitignore

@@ -80,6 +80,7 @@
 /fontconfig/src
 /freetype/src
 /fuchsia-sdk/images
+/fuchsia-sdk/images-internal
 /fuchsia-sdk/sdk
 /gles2_conform
 /glfw/src