Replace hermetic Xcode installation with CIPD-based flow.

The 'API' to the scripts remains the same, only the delivery mechanism changes.

BUG=797051
R=erikchen@chromium.org, justincohen@chromium.org

Change-Id: I8ee5486b107061f9fb6e64354463ac51de53d4cc
Reviewed-on: https://chromium-review.googlesource.com/887819Reviewed-by: John Budorick <jbudorick@chromium.org>
Reviewed-by: Erik Chen <erikchen@chromium.org>
Commit-Queue: Sergey Berezin <sergeyberezin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#553871}

build/mac/should_use_hermetic_xcode.py

@@ -30,7 +30,7 @@ def _IsCorpMachine():
 def main():
   allow_corp = sys.argv[1] == 'mac' and _IsCorpMachine()
   if os.environ.get('FORCE_MAC_TOOLCHAIN') or allow_corp:
-    if not mac_toolchain.PlatformMeetsHermeticXcodeRequirements(sys.argv[1]):
+    if not mac_toolchain.PlatformMeetsHermeticXcodeRequirements():
       return "2"
     return "1"
   else:

build/mac_toolchain.py

 #!/usr/bin/env python
-# Copyright 2016 The Chromium Authors. All rights reserved.
+# Copyright 2018 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -7,40 +7,31 @@
 If should_use_hermetic_xcode.py emits "1", and the current toolchain is out of
 date:
   * Downloads the hermetic mac toolchain
-    * Requires gsutil to be configured.
+    * Requires CIPD authentication. Run `cipd auth-login`, use Google account.
   * Accepts the license.
     * If xcode-select and xcodebuild are not passwordless in sudoers, requires
       user interaction.
 
-The toolchain version can be overridden by setting IOS_TOOLCHAIN_REVISION or
-MAC_TOOLCHAIN_REVISION with the full revision, e.g. 9A235-1.
+The toolchain version can be overridden by setting MAC_TOOLCHAIN_REVISION with
+the full revision, e.g. 9A235.
 """
 
-from distutils.version import LooseVersion
 import os
 import platform
-import plistlib
 import shutil
 import subprocess
 import sys
-import tarfile
-import time
-import tempfile
-import urllib2
 
-# This can be changed after running /build/package_mac_toolchain.py.
+
+# This can be changed after running:
+#    mac_toolchain upload -xcode-path path/to/Xcode.app
 MAC_TOOLCHAIN_VERSION = '8E2002'
-MAC_TOOLCHAIN_SUB_REVISION = 3
-MAC_TOOLCHAIN_VERSION = '%s-%s' % (MAC_TOOLCHAIN_VERSION,
-                                   MAC_TOOLCHAIN_SUB_REVISION)
+
 # The toolchain will not be downloaded if the minimum OS version is not met.
 # 16 is the major version number for macOS 10.12.
 MAC_MINIMUM_OS_VERSION = 16
 
-IOS_TOOLCHAIN_VERSION = '9C40b'
-IOS_TOOLCHAIN_SUB_REVISION = 1
-IOS_TOOLCHAIN_VERSION = '%s-%s' % (IOS_TOOLCHAIN_VERSION,
-                                   IOS_TOOLCHAIN_SUB_REVISION)
+MAC_TOOLCHAIN_INSTALLER = 'mac_toolchain'
 
 # Absolute path to src/ directory.
 REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -49,226 +40,121 @@ REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 GCLIENT_CONFIG = os.path.join(os.path.dirname(REPO_ROOT), '.gclient')
 
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
-TOOLCHAIN_BUILD_DIR = os.path.join(BASE_DIR, '%s_files', 'Xcode.app')
-STAMP_FILE = os.path.join(BASE_DIR, '%s_files', 'toolchain_build_revision')
-TOOLCHAIN_URL = 'gs://chrome-mac-sdk/'
+TOOLCHAIN_ROOT = os.path.join(BASE_DIR, 'mac_files')
+TOOLCHAIN_BUILD_DIR = os.path.join(TOOLCHAIN_ROOT, 'Xcode.app')
+STAMP_FILE = os.path.join(TOOLCHAIN_ROOT, 'toolchain_build_revision')
 
 
-def PlatformMeetsHermeticXcodeRequirements(target_os):
-  if target_os == 'ios':
-    return True
+def PlatformMeetsHermeticXcodeRequirements():
   return int(platform.release().split('.')[0]) >= MAC_MINIMUM_OS_VERSION
 
 
-def GetPlatforms():
-  target_os = set(['mac'])
-  try:
-    env = {}
-    execfile(GCLIENT_CONFIG, env, env)
-    target_os |= set(env.get('target_os', target_os))
-  except:
-    pass
-  return target_os
-
-
-def ReadStampFile(target_os):
-  """Return the contents of the stamp file, or '' if it doesn't exist."""
-  try:
-    with open(STAMP_FILE % target_os, 'r') as f:
-      return f.read().rstrip()
-  except IOError:
-    return ''
-
-
-def WriteStampFile(target_os, s):
-  """Write s to the stamp file."""
-  EnsureDirExists(os.path.dirname(STAMP_FILE % target_os))
-  with open(STAMP_FILE % target_os, 'w') as f:
-    f.write(s)
-    f.write('\n')
-
-
-def EnsureDirExists(path):
-  if not os.path.exists(path):
-    os.makedirs(path)
-
-
-def DownloadAndUnpack(url, output_dir):
-  """Decompresses |url| into a cleared |output_dir|."""
-  temp_name = tempfile.mktemp(prefix='mac_toolchain')
-  try:
-    print 'Downloading new toolchain.'
-    subprocess.check_call(['gsutil.py', 'cp', url, temp_name])
-    if os.path.exists(output_dir):
-      print 'Deleting old toolchain.'
-      shutil.rmtree(output_dir)
-    EnsureDirExists(output_dir)
-    print 'Unpacking new toolchain.'
-    tarfile.open(mode='r:gz', name=temp_name).extractall(path=output_dir)
-  finally:
-    if os.path.exists(temp_name):
-      os.unlink(temp_name)
-
-
-def CanAccessToolchainBucket():
-  """Checks whether the user has access to |TOOLCHAIN_URL|."""
-  proc = subprocess.Popen(['gsutil.py', 'ls', TOOLCHAIN_URL],
-                           stdout=subprocess.PIPE)
-  proc.communicate()
-  return proc.returncode == 0
-
-
-def LoadPlist(path):
-  """Loads Plist at |path| and returns it as a dictionary."""
-  fd, name = tempfile.mkstemp()
-  try:
-    subprocess.check_call(['plutil', '-convert', 'xml1', '-o', name, path])
-    with os.fdopen(fd, 'r') as f:
-      return plistlib.readPlist(f)
-  finally:
-    os.unlink(name)
-
-
-def FinalizeUnpack(output_dir, target_os):
-  """Use xcodebuild to accept new toolchain license and run first launch
-  installers if necessary.  Don't accept the license if a newer license has
-  already been accepted. This only works if xcodebuild and xcode-select are
-  passwordless in sudoers."""
-
-  # Check old license
-  try:
-    target_license_plist_path = os.path.join(
-        output_dir, 'Contents','Resources','LicenseInfo.plist')
-    target_license_plist = LoadPlist(target_license_plist_path)
-    build_type = target_license_plist['licenseType']
-    build_version = target_license_plist['licenseID']
-
-    accepted_license_plist = LoadPlist(
-        '/Library/Preferences/com.apple.dt.Xcode.plist')
-    agreed_to_key = 'IDELast%sLicenseAgreedTo' % build_type
-    last_license_agreed_to = accepted_license_plist[agreed_to_key]
-
-    # Historically all Xcode build numbers have been in the format of AANNNN, so
-    # a simple string compare works.  If Xcode's build numbers change this may
-    # need a more complex compare.
-    if build_version <= last_license_agreed_to:
-      # Don't accept the license of older toolchain builds, this will break the
-      # license of newer builds.
-      return
-  except (subprocess.CalledProcessError, KeyError):
-    # If there's never been a license of type |build_type| accepted,
-    # |target_license_plist_path| or |agreed_to_key| may not exist.
-    pass
-
-  print "Accepting license."
-  target_version_plist_path = os.path.join(
-      output_dir, 'Contents','version.plist')
-  target_version_plist = LoadPlist(target_version_plist_path)
-  short_version_string = target_version_plist['CFBundleShortVersionString']
-  old_path = subprocess.Popen(['/usr/bin/xcode-select', '-p'],
-                               stdout=subprocess.PIPE).communicate()[0].strip()
-  try:
-    build_dir = os.path.join(output_dir, 'Contents/Developer')
-    subprocess.check_call(['sudo', '/usr/bin/xcode-select', '-s', build_dir])
-    subprocess.check_call(['sudo', '/usr/bin/xcodebuild', '-license', 'accept'])
-
-    if target_os == 'ios' and \
-        LooseVersion(short_version_string) >= LooseVersion("9.0"):
-      print "Installing packages."
-      subprocess.check_call(['sudo', '/usr/bin/xcodebuild', '-runFirstLaunch'])
-  finally:
-    subprocess.check_call(['sudo', '/usr/bin/xcode-select', '-s', old_path])
-
-
-def _UseHermeticToolchain(target_os):
+def _UseHermeticToolchain():
   current_dir = os.path.dirname(os.path.realpath(__file__))
   script_path = os.path.join(current_dir, 'mac/should_use_hermetic_xcode.py')
-  proc = subprocess.Popen([script_path, target_os], stdout=subprocess.PIPE)
+  proc = subprocess.Popen([script_path, 'mac'], stdout=subprocess.PIPE)
   return '1' in proc.stdout.readline()
 
 
-def RequestGsAuthentication():
-  """Requests that the user authenticate to be able to access gs://.
-  """
-  print 'Access to ' + TOOLCHAIN_URL + ' not configured.'
+def RequestCipdAuthentication():
+  """Requests that the user authenticate to access Xcode CIPD packages."""
+
+  print 'Access to Xcode CIPD package requires authentication.'
   print '-----------------------------------------------------------------'
   print
   print 'You appear to be a Googler.'
   print
-  print 'I\'m sorry for the hassle, but you need to do a one-time manual'
+  print 'I\'m sorry for the hassle, but you may need to do a one-time manual'
   print 'authentication. Please run:'
   print
-  print '    download_from_google_storage --config'
+  print '    cipd auth-login'
   print
   print 'and follow the instructions.'
   print
-  print 'NOTE 1: Use your google.com credentials, not chromium.org.'
-  print 'NOTE 2: Enter 0 when asked for a "project-id".'
+  print 'NOTE: Use your google.com credentials, not chromium.org.'
   print
   print '-----------------------------------------------------------------'
   print
   sys.stdout.flush()
-  sys.exit(1)
 
 
-def DownloadHermeticBuild(target_os, toolchain_version, toolchain_filename):
-  if not _UseHermeticToolchain(target_os):
-    return 0
+def PrintError(message):
+  # Flush buffers to ensure correct output ordering.
+  sys.stdout.flush()
+  sys.stderr.write(message + '\n')
+  sys.stderr.flush()
 
-  toolchain_output_path = TOOLCHAIN_BUILD_DIR % target_os
-  if ReadStampFile(target_os) == toolchain_version:
-    FinalizeUnpack(toolchain_output_path, target_os)
-    return 0
 
-  if not CanAccessToolchainBucket():
-    RequestGsAuthentication()
-    return 1
+def InstallXcode(xcode_build_version, installer_cmd, xcode_app_path):
+  """Installs the requested Xcode build version.
 
-  # Reset the stamp file in case the build is unsuccessful.
-  WriteStampFile(target_os, '')
+  Args:
+    xcode_build_version: (string) Xcode build version to install.
+    installer_cmd: (string) Path to mac_toolchain command to install Xcode.
+      See https://chromium.googlesource.com/infra/infra/+/master/go/src/infra/cmd/mac_toolchain/
+    xcode_app_path: (string) Path to install the contents of Xcode.app.
 
-  toolchain_file = '%s.tgz' % toolchain_version
-  toolchain_full_url = TOOLCHAIN_URL + toolchain_file
+  Returns:
+    True if installation was successful. False otherwise.
+  """
+  args = [
+      installer_cmd, 'install',
+      '-kind', 'mac',
+      '-xcode-version', xcode_build_version.lower(),
+      '-output-dir', xcode_app_path,
+  ]
 
-  print 'Updating toolchain to %s...' % toolchain_version
   try:
-    toolchain_file = toolchain_filename % toolchain_version
-    toolchain_full_url = TOOLCHAIN_URL + toolchain_file
-    DownloadAndUnpack(toolchain_full_url, toolchain_output_path)
-    FinalizeUnpack(toolchain_output_path, target_os)
+    subprocess.check_call(args)
+  except subprocess.CalledProcessError as e:
+    PrintError('Xcode build version %s failed to install: %s\n' % (
+        xcode_build_version, e))
+    RequestCipdAuthentication()
+    return False
+  except OSError as e:
+    PrintError(('Xcode installer "%s" failed to execute'
+                ' (not on PATH or not installed).') % installer_cmd)
+    return False
 
-    print 'Toolchain %s unpacked.' % toolchain_version
-    WriteStampFile(target_os, toolchain_version)
-    return 0
-  except Exception as e:
-    print 'Failed to download toolchain %s.' % toolchain_file
-    print 'Exception %s' % e
-    print 'Exiting.'
-    return 1
+  return True
 
 
 def main():
   if sys.platform != 'darwin':
     return 0
 
-  for target_os in GetPlatforms():
-    if not PlatformMeetsHermeticXcodeRequirements(target_os):
-      print 'OS version does not support toolchain.'
-      continue
-
-    if target_os == 'ios':
-      toolchain_version = os.environ.get('IOS_TOOLCHAIN_REVISION',
-                                          IOS_TOOLCHAIN_VERSION)
-      toolchain_filename = 'ios-toolchain-%s.tgz'
-    else:
-      toolchain_version = os.environ.get('MAC_TOOLCHAIN_REVISION',
-                                          MAC_TOOLCHAIN_VERSION)
-      toolchain_filename = 'toolchain-%s.tgz'
-
-    return_value = DownloadHermeticBuild(
-        target_os, toolchain_version, toolchain_filename)
-    if return_value:
-      return return_value
+  if not _UseHermeticToolchain():
+    print 'Skipping Mac toolchain installation for mac'
+    return 0
+
+  if not PlatformMeetsHermeticXcodeRequirements():
+    print 'OS version does not support toolchain.'
+    return 0
+
+  toolchain_version = os.environ.get('MAC_TOOLCHAIN_REVISION',
+                                      MAC_TOOLCHAIN_VERSION)
+
+  # On developer machines, mac_toolchain tool is provided by
+  # depot_tools. On the bots, the recipe is responsible for installing
+  # it and providing the path to the executable.
+  installer_cmd = os.environ.get('MAC_TOOLCHAIN_INSTALLER',
+                                 MAC_TOOLCHAIN_INSTALLER)
+
+  toolchain_root = TOOLCHAIN_ROOT
+  xcode_app_path = TOOLCHAIN_BUILD_DIR
+  stamp_file = STAMP_FILE
+
+  # Delete the old "hermetic" installation if detected.
+  # TODO(crbug.com/797051): remove this once the old "hermetic" solution is no
+  # longer in use.
+  if os.path.exists(stamp_file):
+    print 'Detected old hermetic installation at %s. Deleting.' % (
+      toolchain_root)
+    shutil.rmtree(toolchain_root)
+
+  success = InstallXcode(toolchain_version, installer_cmd, xcode_app_path)
+  if not success:
+    return 1
 
   return 0