[Fuchsia] Add Mac build support.

This adds support to build Chromium for Fuchsia on macOS. Currently,
this configuration is to be treated as best-effort.

CQ_INCLUDE_TRYBOTS=luci.chromium.try:fuchsia_arm64_cast_audio;luci.chromium.try:fuchsia_x64_cast_audio

Bug: 707030
Test: Locally, builds on mac.
Change-Id: I9e4bde1b7ff658f51586856ae80598c93a2b2e33
Reviewed-on: https://chromium-review.googlesource.com/1185020Reviewed-by: John Budorick <jbudorick@chromium.org>
Reviewed-by: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Wez <wez@chromium.org>
Commit-Queue: Fabrice de Gans-Riberi <fdegans@chromium.org>
Cr-Commit-Position: refs/heads/master@{#587546}

DEPS

@@ -976,14 +976,25 @@ deps = {
   'src/third_party/pywebsocket/src':
     Var('chromium_git') + '/external/github.com/google/pywebsocket.git' + '@' + '2d7b73c3acbd0f41dcab487ae5c97c6feae06ce2',
 
-  'src/third_party/qemu': {
+  'src/third_party/qemu-linux-x64': {
       'packages': [
           {
               'package': 'fuchsia/qemu/linux-amd64',
               'version': '9cc486c5b18a0be515c39a280ca9a309c54cf994'
           },
       ],
-      'condition': 'checkout_fuchsia',
+      'condition': 'host_os == "linux" and checkout_fuchsia',
+      'dep_type': 'cipd',
+  },
+
+  'src/third_party/qemu-mac-x64': {
+      'packages': [
+          {
+              'package': 'fuchsia/qemu/mac-amd64',
+              'version': '2d3358ae9a569b2d4a474f498b32b202a152134f'
+          },
+      ],
+      'condition': 'host_os == "mac" and checkout_fuchsia',
       'dep_type': 'cipd',
   },
 
@@ -1986,11 +1997,11 @@ hooks = [
   },
   {
     # Mac doesn't use lld so it's not included in the default clang bundle
-    # there.  lld is however needed in win cross builds, so download it there.
-    # Should run after the clang hook.
+    # there.  lld is however needed in win and Fuchsia cross builds, so
+    # download it there. Should run after the clang hook.
     'name': 'lld/mac',
     'pattern': '.',
-    'condition': 'host_os == "mac" and checkout_win',
+    'condition': 'host_os == "mac" and (checkout_win or checkout_fuchsia)',
     'action': ['python', 'src/tools/clang/scripts/download_lld_mac.py'],
   },
   {

build/config/fuchsia/BUILD.gn

@@ -121,7 +121,7 @@ action("blobstore_extended_qcow2") {
   ]
 
   args = [
-    rebase_path("//third_party/qemu/bin/qemu-img", root_build_dir),
+    rebase_path("${qemu_root}/bin/qemu-img", root_build_dir),
     "convert",
     "-f",
     "raw",

build/config/fuchsia/config.gni

@@ -17,3 +17,6 @@ if (current_cpu == "arm64") {
 } else {
   assert(false, "No libraries available for architecture: $current_cpu")
 }
+
+# Compute the qemu path.
+qemu_root = "//third_party/qemu-${host_os}-${host_cpu}"

build/config/fuchsia/rules.gni

@@ -69,7 +69,7 @@ template("fuchsia_package_runner") {
       _manifest_path,
       "//build/fuchsia/",
       "//build/util/lib/",
-      "//third_party/qemu",
+      "${qemu_root}/",
       "${fuchsia_sdk}/",
     ]
 

build/fuchsia/boot_data.py

@@ -85,7 +85,7 @@ def _MakeQcowDisk(output_dir, disk_path):
   """Creates a QEMU copy-on-write version of |disk_path| in the output
   directory."""
 
-  qimg_path = os.path.join(common.QEMU_ROOT, 'bin', 'qemu-img')
+  qimg_path = os.path.join(common.GetQemuRootForPlatform(), 'bin', 'qemu-img')
   output_path = os.path.join(output_dir,
                              os.path.basename(disk_path) + '.qcow2')
   subprocess.check_call([qimg_path, 'create', '-q', '-f', 'qcow2',

build/fuchsia/common.py

@@ -3,10 +3,11 @@
 # found in the LICENSE file.
 
 import os
+import platform
+import sys
 
 DIR_SOURCE_ROOT = os.path.abspath(
     os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
-QEMU_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'qemu')
 SDK_ROOT = os.path.join(DIR_SOURCE_ROOT, 'third_party', 'fuchsia-sdk', 'sdk')
 
 def EnsurePathExists(path):
@@ -17,3 +18,24 @@ def EnsurePathExists(path):
     raise IOError('Missing file: ' + path)
 
   return path
+
+def GetHostOsFromPlatform():
+  host_platform = sys.platform
+  if host_platform.startswith('linux'):
+    return 'linux'
+  elif host_platform.startswith('darwin'):
+    return 'mac'
+  raise Exception('Unsupported host platform: %s' % host_platform)
+
+def GetHostArchFromPlatform():
+  host_arch = platform.machine()
+  if host_arch == 'x86_64':
+    return 'x64'
+  elif host_arch == 'aarch64':
+    return 'arm64'
+  raise Exception('Unsupported host architecture: %s' % host_arch)
+
+def GetQemuRootForPlatform():
+  return os.path.join(DIR_SOURCE_ROOT, 'third_party',
+                      'qemu-' + GetHostOsFromPlatform() + '-' +
+                       GetHostArchFromPlatform())

build/fuchsia/mac.sdk.sha1

+61465d3f598a8ad7e6cde3f6505684c63d5361de
\ No newline at end of file

build/fuchsia/qemu_target.py

@@ -14,7 +14,7 @@ import subprocess
 import sys
 import time
 
-from common import QEMU_ROOT, EnsurePathExists
+from common import GetQemuRootForPlatform, EnsurePathExists
 
 
 # Virtual networking configuration data for QEMU.
@@ -57,7 +57,7 @@ class QemuTarget(target.Target):
       self._qemu_process.kill()
 
   def Start(self):
-    qemu_path = os.path.join(QEMU_ROOT, 'bin',
+    qemu_path = os.path.join(GetQemuRootForPlatform(), 'bin',
                              'qemu-system-' + self._GetTargetSdkLegacyArch())
     kernel_args = boot_data.GetKernelArgs(self._output_dir)
 
@@ -100,24 +100,27 @@ class QemuTarget(target.Target):
       ]
 
     # Configure the machine & CPU to emulate, based on the target architecture.
-    # Enable lightweight virtualization (KVM) if the host and guest OS run on
-    # the same architecture.
     if self._target_cpu == 'arm64':
       qemu_command.extend([
           '-machine','virt',
           '-cpu', 'cortex-a53',
       ])
       netdev_type = 'virtio-net-pci'
-      if platform.machine() == 'aarch64':
-        qemu_command.append('-enable-kvm')
     else:
       qemu_command.extend([
           '-machine', 'q35',
-          '-cpu', 'host,migratable=no',
       ])
       netdev_type = 'e1000'
-      if platform.machine() == 'x86_64':
+
+    # On Linux, enable lightweight virtualization (KVM) if the host and guest
+    # architectures are the same.
+    if sys.platform.startswith('linux'):
+      if self._target_cpu == 'arm64' and platform.machine() == 'aarch64':
         qemu_command.append('-enable-kvm')
+      elif self._target_cpu == 'x64' and platform.machine() == 'x86_64':
+        qemu_command.extend([
+            '-enable-kvm', '-cpu', 'host,migratable=no',
+        ])
 
     # Configure virtual network. It is used in the tests to connect to
     # testserver running on the host.

build/fuchsia/run_package.py

@@ -10,12 +10,13 @@ import json
 import logging
 import multiprocessing
 import os
+import select
 import shutil
 import subprocess
+import sys
 import tempfile
 import threading
 import uuid
-import select
 
 from symbolizer import FilterStream
 
@@ -40,47 +41,33 @@ def _ReadMergedLines(streams):
   signals EOF. Absolute output ordering is not guaranteed."""
 
   assert len(streams) > 0
-  poll = select.poll()
   streams_by_fd = {}
   primary_fd = streams[0].fileno()
   for s in streams:
-    poll.register(s.fileno(), select.POLLIN)
     streams_by_fd[s.fileno()] = s
 
-  try:
-    while primary_fd != None:
-      events = poll.poll(1)
-      for fileno, event in events:
-        if event & select.POLLIN:
-          yield streams_by_fd[fileno].readline()
-
-        elif event & select.POLLHUP:
-          poll.unregister(fileno)
-          del streams_by_fd[fileno]
-
-          if fileno == primary_fd:
-            primary_fd = None
-  finally:
-    for fd_to_cleanup, _ in streams_by_fd.iteritems():
-      poll.unregister(fd_to_cleanup)
+  while primary_fd != None:
+    rlist, _, _ = select.select(streams_by_fd, [], [], 0.1)
+    for fileno in rlist:
+      line = streams_by_fd[fileno].readline()
+      if line:
+        yield line
+      elif fileno == primary_fd:
+        primary_fd = None
+      else:
+        del streams_by_fd[fileno]
 
 
 def DrainStreamToStdout(stream, quit_event):
   """Outputs the contents of |stream| until |quit_event| is set."""
 
-  poll = select.poll()
-  poll.register(stream.fileno(), select.POLLIN)
-  try:
-    while not quit_event.is_set():
-      events = poll.poll(1)
-      for fileno, event in events:
-        if event & select.POLLIN:
-          print stream.readline().rstrip()
-        elif event & select.POLLHUP:
-          break
-
-  finally:
-    poll.unregister(stream.fileno())
+  while not quit_event.is_set():
+    rlist, _, _ = select.select([ stream ], [], [], 0.1)
+    if rlist:
+      line = rlist[0].readline()
+      if not line:
+        return
+      print line.rstrip()
 
 
 def RunPackage(output_dir, target, package_path, package_name, package_deps,

build/fuchsia/update_sdk.py

@@ -13,7 +13,7 @@ import sys
 import tarfile
 import tempfile
 
-SDK_HASH_FILE = os.path.join(os.path.dirname(__file__), 'sdk.sha1')
+from common import GetHostOsFromPlatform, GetHostArchFromPlatform
 
 REPOSITORY_ROOT = os.path.abspath(os.path.join(
     os.path.dirname(__file__), '..', '..'))
@@ -24,6 +24,16 @@ import find_depot_tools
 SDK_SUBDIRS = ["arch", "pkg", "qemu", "sysroot", "target",
                "toolchain_libs", "tools"]
 
+def GetSdkHashForPlatform():
+  if sys.platform.startswith('darwin'):
+    return os.path.join(os.path.dirname(__file__), 'mac.sdk.sha1')
+  else:
+    return os.path.join(os.path.dirname(__file__), 'sdk.sha1')
+
+def GetBucketForPlatform():
+  return 'gs://fuchsia/sdk/{platform}-amd64/'.format(
+      platform = GetHostOsFromPlatform())
+
 
 def EnsureDirExists(path):
   if not os.path.exists(path):
@@ -63,11 +73,12 @@ def main():
   # there.
   Cleanup(os.path.join(REPOSITORY_ROOT, 'third_party', 'fuchsia-sdk'))
 
-  with open(SDK_HASH_FILE, 'r') as f:
+  hash_file = GetSdkHashForPlatform()
+  with open(hash_file, 'r') as f:
     sdk_hash = f.read().strip()
 
   if not sdk_hash:
-    print >>sys.stderr, 'No SHA1 found in %s' % SDK_HASH_FILE
+    print >>sys.stderr, 'No SHA1 found in %s' % hash_file
     return 1
 
   output_dir = os.path.join(REPOSITORY_ROOT, 'third_party', 'fuchsia-sdk',
@@ -89,9 +100,8 @@ def main():
   os.close(fd)
 
   try:
-    bucket = 'gs://fuchsia/sdk/linux-amd64/'
     cmd = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'),
-           'cp', bucket + sdk_hash, tmp]
+           'cp', GetBucketForPlatform() + sdk_hash, tmp]
     subprocess.check_call(cmd)
     with open(tmp, 'rb') as f:
       EnsureDirExists(output_dir)

build/toolchain/fuchsia/BUILD.gn

@@ -9,14 +9,17 @@ import("//build/config/fuchsia/config.gni")
 # the different target architectures.
 template("fuchsia_clang_toolchain") {
   clang_toolchain(target_name) {
-    assert(host_os == "linux")
+    assert(host_os == "linux" || host_os == "mac")
     assert(defined(invoker.toolchain_args),
            "toolchain_args must be defined for fuchsia_clang_toolchain()")
 
     # We want to build and strip binaries, but retain the unstripped binaries
     # in runtime_deps to make them available for isolates.
-    strip = rebase_path("//third_party/eu-strip/bin/eu-strip", root_build_dir)
-    use_unstripped_as_runtime_outputs = true
+    # TODO(https://crbug.com/877080): Switch to llvm-strip.
+    if (host_os == "linux") {
+      strip = rebase_path("//third_party/eu-strip/bin/eu-strip", root_build_dir)
+      use_unstripped_as_runtime_outputs = true
+    }
 
     toolchain_args = invoker.toolchain_args
     toolchain_args.current_os = "fuchsia"

docs/fuchsia_build_instructions.md

@@ -13,7 +13,8 @@ There are instructions for other platforms linked from the
 *   At least 100GB of free disk space.
 *   You must have Git and Python installed already.
 
-Most development is done on Ubuntu.
+Most development is done on Ubuntu. Mac build is supported on a best-effort
+basis.
 
 ## Install `depot_tools`
 
@@ -82,6 +83,16 @@ The remaining instructions assume you have switched to the `src` directory:
 $ cd src
 ```
 
+## (Mac-only) Download additional required Clang binaries
+
+Go to [this page](https://chrome-infra-packages.appspot.com/p/fuchsia/clang/mac-amd64/+/)
+and download the most recent build. Extract `bin/llvm-ar` to the clang folder
+in Chromium:
+
+```shell
+$ unzip /path/to/clang.zip bin/llvm-ar -d ${CHROMIUM_SRC}/third_party/llvm-build/Release+Asserts
+```
+
 ## Setting up the build
 
 Chromium uses [Ninja](https://ninja-build.org) as its main build tool along with

third_party/.gitignore

@@ -179,7 +179,8 @@
 /python_26
 /pywebsocket/src
 /pywebsocket/src
-/qemu
+/qemu-linux-x64
+/qemu-mac-x64
 /quic_trace/src
 /r8/lib
 /re2/src

webrunner/BUILD.gn

@@ -222,6 +222,13 @@ fidl_library("fidl") {
   ]
 }
 
+# gn binary location.
+if (host_os == "mac") {
+  _gn_path = "//buildtools/mac/gn"
+} else if (host_os == "linux") {
+  _gn_path = "//buildtools/linux64/gn"
+}
+
 # Build location where all Fuchsia archive source files are placed.
 _artifact_root = "$root_out_dir/fuchsia_artifacts"
 
@@ -230,7 +237,7 @@ _license_path = "$_artifact_root/LICENSE"
 action("license") {
   script = "//tools/licenses.py"
   inputs = [
-    "//buildtools/linux64/gn",
+    "$_gn_path",
   ]
   outputs = [
     _license_path,