Compile on macOS using binaries from Xcode, rather than a full install.

Previously, Chromium used a hermetic toolchain. This was basically a full
install of Xcode, which has OS version dependencies. This CL keeps the hermetic
toolchain, since it's necessary for clang + code coverage.

Compilation of Chrome now uses a separate package of binaries from
Xcode. This consists of a whitelist of the OS-version agnostic binaries from
Xcode. These binaries do not have [relevant] OS version dependencies.

Change-Id: Ie6f30b4b6b3ccd4a7538feb73d6828cfa4a02891
Bug: 965663
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1673572Reviewed-by: Nico Weber <thakis@chromium.org>
Commit-Queue: Erik Chen <erikchen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#678463}

build/mac/find_sdk.py

@@ -84,11 +84,7 @@ def main():
   sdk_dir = os.path.join(
       dev_dir, 'Platforms/MacOSX.platform/Developer/SDKs')
 
-  # Xcode must be installed, its license agreement must be accepted, and its
-  # command-line tools must be installed. Stand-alone installations (in
-  # /Library/Developer/CommandLineTools) are not supported.
-  # https://bugs.chromium.org/p/chromium/issues/detail?id=729990#c1
-  if not os.path.isdir(sdk_dir) or not '.app/Contents/Developer' in sdk_dir:
+  if not os.path.isdir(sdk_dir):
     raise SdkError('Install Xcode, launch it, accept the license ' +
       'agreement, and run `sudo xcode-select -s /path/to/Xcode.app` ' +
       'to continue.')

build/mac_toolchain.py

@@ -11,6 +11,7 @@ date:
   * Accepts the license.
     * If xcode-select and xcodebuild are not passwordless in sudoers, requires
       user interaction.
+  * Downloads standalone binaries from [a possibly different version of Xcode].
 
 The toolchain version can be overridden by setting MAC_TOOLCHAIN_REVISION with
 the full revision, e.g. 9A235.
@@ -19,7 +20,9 @@ the full revision, e.g. 9A235.
 from __future__ import print_function
 
 import os
+import pkg_resources
 import platform
+import plistlib
 import shutil
 import subprocess
 import sys
@@ -27,8 +30,19 @@ import sys
 
 # This can be changed after running:
 #    mac_toolchain upload -xcode-path path/to/Xcode.app
+# The hermetic install of Xcode is used:
+#  1) For sizes support
+#  2) To build clang
+#  3) For code-coverage support.
+# These should eventually be phased out to use the new deployment of
+# xcode_binaries, see InstallXcodeBinaries. https://crbug.com/984746
 MAC_TOOLCHAIN_VERSION = '9E501'
 
+# This contains binaries from Xcode 9.3.1, along with the 10.13 SDK.
+# To build this package, see comments in build/xcode_binaries.yaml
+MAC_BINARIES_LABEL = 'infra_internal/ios/xcode/xcode_binaries/mac-amd64'
+MAC_BINARIES_TAG = 'oXeIQqw4E0pXIStoixHwKg_1TJZTJGhfhmn2fq8LPOAC'
+
 # The toolchain will not be downloaded if the minimum OS version is not met.
 # 17 is the major version number for macOS 10.13.
 # 9E145 (Xcode 9.3) only runs on 10.13.2 and newer.
@@ -129,6 +143,69 @@ def InstallXcode(xcode_build_version, installer_cmd, xcode_app_path):
   return True
 
 
+def InstallXcodeBinaries():
+  """Installs the Xcode binaries needed to build Chrome and accepts the license.
+
+  This is the replacement for InstallXcode that installs a trimmed down version
+  of Xcode that is OS-version agnostic.
+  """
+  # First make sure the directory exists. It will serve as the cipd root. This
+  # also ensures that there will be no conflicts of cipd root.
+  binaries_root = os.path.join(TOOLCHAIN_ROOT, 'xcode_binaries')
+  if not os.path.exists(binaries_root):
+    os.mkdir(binaries_root)
+
+  # 'cipd ensure' is idempotent.
+  args = [
+      'cipd', 'ensure', '-root', binaries_root, '-ensure-file', '-'
+  ]
+  p = subprocess.Popen(args, stdin=subprocess.PIPE)
+  p.communicate(input=MAC_BINARIES_LABEL + ' ' + MAC_BINARIES_TAG)
+
+  # Accept the license for this version of Xcode if it's newer than the
+  # currently accepted version.
+  cipd_xcode_version_plist_path = os.path.join(
+      binaries_root, 'Contents/version.plist')
+  cipd_xcode_version_plist = plistlib.readPlist(cipd_xcode_version_plist_path)
+  cipd_xcode_version = cipd_xcode_version_plist['CFBundleShortVersionString']
+
+  cipd_license_path = os.path.join(
+      binaries_root, 'Contents/Resources/LicenseInfo.plist')
+  cipd_license_plist = plistlib.readPlist(cipd_license_path)
+  cipd_license_version = cipd_license_plist['licenseID']
+
+  should_overwrite_license = True
+  current_license_path = '/Library/Preferences/com.apple.dt.Xcode.plist'
+  if os.path.exists(current_license_path):
+    current_license_plist = plistlib.readPlist(current_license_path)
+    xcode_version = current_license_plist['IDEXcodeVersionForAgreedToGMLicense']
+    if (pkg_resources.parse_version(xcode_version) >=
+        pkg_resources.parse_version(cipd_xcode_version)):
+      should_overwrite_license = False
+
+  if not should_overwrite_license:
+    return
+
+  # Use puppet's sudoers script to accept the license if its available.
+  license_accept_script = '/usr/local/bin/xcode_accept_license.py'
+  if os.path.exists(license_accept_script):
+    args = ['sudo', license_accept_script, '--xcode_version',
+            cipd_xcode_version, '--license-version', cipd_license_version]
+    subprocess.call(args)
+    return
+
+  # Otherwise manually accept the license. This will prompt for sudo.
+  print('Accepting new Xcode license.')
+  args = ['sudo', 'defaults', 'write', current_license_path,
+          'IDEXcodeVersionForAgreedToGMLicense', cipd_xcode_version]
+  subprocess.call(args)
+  args = ['sudo', 'defaults', 'write', current_license_path,
+          'IDELastGMLicenseAgreedTo', cipd_license_version]
+  subprocess.call(args)
+  args = ['sudo', 'plutil', '-convert', 'xml1', current_license_path]
+  subprocess.call(args)
+
+
 def main():
   if sys.platform != 'darwin':
     return 0
@@ -166,6 +243,8 @@ def main():
   if not success:
     return 1
 
+  InstallXcodeBinaries()
+
   return 0
 
 

build/toolchain/toolchain.gni

@@ -36,8 +36,14 @@ if (generate_linker_map) {
 
 # The path to the hermetic install of Xcode. Only relevant when
 # use_system_xcode = false.
-hermetic_xcode_path =
-    rebase_path("//build/${target_os}_files/Xcode.app", "", root_build_dir)
+if (is_ios) {
+  hermetic_xcode_path =
+      rebase_path("//build/${target_os}_files/Xcode.app", "", root_build_dir)
+} else {
+  hermetic_xcode_path = rebase_path("//build/${target_os}_files/xcode_binaries",
+                                    "",
+                                    root_build_dir)
+}
 
 declare_args() {
   if (is_clang) {

build/xcode_binaries.yaml

+# This yaml file is used to package binaries from Xcode.app.
+# To use this:
+#   1) Move Xcode.app to the same directory as this file.
+#   2) Rename Xcode.app to xcode_binaries
+#   3) Call `cipd create --pkg-def xcode_binaries.yaml`
+# To deploy the newly created cipd package across the fleet, modify
+# mac_toolchain.py to point to the new cipd hash.
+#
+# The ACLs for this package are determined by the directory structure. The
+# nomenclature mirrors that of the hermetic toolchain to avoid ACL duplication.
+package: infra_internal/ios/xcode/xcode_binaries/mac-amd64
+description: A hermetic deployment of all Xcode binaries used to build Chromium.
+root: "xcode_binaries"
+data:
+  - dir: Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/
+  - dir: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/share/bison
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/bison
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/gm4
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/gperf
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/llvm-nm
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/llvm-objdump
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/llvm-otool
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/libtool
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/mig
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/nm
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/otool
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/size
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/strip
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/libtapi.dylib
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/libLTO.dylib
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/libswiftDemangle.dylib
+  - file: Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/libexec/migcom
+  - file: Contents/Resources/English.lproj/License.rtf
+  - file: Contents/Resources/LicenseInfo.plist
+  - file: Contents/version.plist