Add gclient hook to download prebuilt mojo_shell

This adds a gclient hook that downloads prebuilt mojo_shell binaries
from google storage (linux only for now) and gyp/gn flags to toggle
using these prebuilt binaries instead of ones built from source. This
also adds a helper script that uploads a locally built mojo_shell binary
to the appropriate place in google storage.  The versioning scheme is:

*) mojo/public/VERSION contains an identifier (committish currently) for
  the mojo public SDK. The mojo_shell binary must match the interfaces
  defined in this version of mojo/public/.

*) gs://mojo/shell/$VERSION/$PLATFORM.zip contains a prebuilt mojo_shell
  binary for a particular version. $PLATFORM for now must be linux-x64,
  but we can add more later as needed.

gclient runhooks downloads the shell binary referred to by
mojo/public/VERSION if it is available for the host platform to
mojo/public/tools/prebuilt/ and updates mojo/public/tools/prebuilt/VERSION
to tag the version downloaded if successful. On subsequent runs the hook
will check this against mojo/public/VERSION and avoid downloading again if
the binary is up to date.

This also adds variables to the GYP and GN builds that control if building
'mojo' builds a mojo_shell binary from source or copies the prebuilt binary
into the outdir.  They both default to false for now.

R=abarth@chromium.org

Review URL: https://codereview.chromium.org/667853002

Cr-Commit-Position: refs/heads/master@{#300600}

.gitignore

@@ -210,6 +210,7 @@ v8.log
 /mojo/mojo_wm_flow_embedder_bindings.xml
 /mojo/mojom_test.xml
 /mojo/sample_service.xml
+/mojo/public/tools/prebuilt/
 /native_client
 /net/Debug
 /net/net_derived_sources.xml

DEPS

@@ -747,6 +747,12 @@ hooks = [
                 'src/third_party/apache-win32',
     ],
   },
+  # Pull the mojo_shell binary, used for mojo development
+  {
+    'name': 'download_mojo_shell',
+    'pattern': '',
+    'action': [ 'python', 'src/mojo/public/tools/download_shell_binary.py' ],
+  },
   {
     # A change to a .gyp, .gypi, or to GYP itself should run the generator.
     'name': 'gyp',

mojo/BUILD.gn

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 import("//build/config/ui.gni")
+import("//mojo/public/mojo.gni")
 
 group("mojo") {
   # Meta-target, don't link into production code.
@@ -18,10 +19,15 @@ group("mojo") {
     "//mojo/examples",
     "//mojo/public",
     "//mojo/services",
-    "//mojo/shell:mojo_shell",
     "//mojo/tools/package_manager",
   ]
 
+  if (use_prebuilt_mojo_shell) {
+    deps += [ "//mojo/public/tools:copy_mojo_shell" ]
+  } else {
+    deps += [ "//mojo/shell:mojo_shell" ]
+  }
+
   if (is_android) {
     deps += [
       "//mojo/android",

mojo/mojo.gyp

@@ -19,6 +19,9 @@
     'mojo_services.gypi',
     'mojo_variables.gypi',
   ],
+  'variables': {
+    'use_prebuilt_mojo_shell%': 0,
+  },
   'targets': [
     {
       # GN version: //mojo
@@ -212,27 +215,6 @@
         'shell/shell_test_helper.h',
       ],
     },
-    {
-      # GN version: //mojo/shell
-      'target_name': 'mojo_shell',
-      'type': 'executable',
-      'dependencies': [
-        '../base/base.gyp:base',
-        'mojo_base.gyp:mojo_common_lib',
-        'mojo_base.gyp:mojo_environment_chromium',
-        'mojo_shell_lib',
-      ],
-      'sources': [
-        'shell/desktop/mojo_main.cc',
-      ],
-      'conditions': [
-        ['component=="shared_library"', {
-          'dependencies': [
-            '../ui/gfx/gfx.gyp:gfx',
-          ],
-        }],
-      ],
-    },
     {
       # GN version: //mojo/shell:mojo_shell_tests
       'target_name': 'mojo_shell_tests',
@@ -418,6 +400,45 @@
     },
   ],
   'conditions': [
+    ['<(use_prebuilt_mojo_shell)==1', {
+      'targets': [
+        {
+          # GN version: //mojo/public/tools:mojo_shell
+          'target_name': 'mojo_shell',
+          'type': 'none',
+          'copies': [{
+            'destination': '<(PRODUCT_DIR)',
+            'files': [
+              'public/tools/prebuilt/mojo_shell',
+            ],
+          }],
+        },
+      ]
+    }, {  # use_prebuilt_mojo_shell != 1
+      'targets': [
+        {
+          # GN version: //mojo/shell
+          'target_name': 'mojo_shell',
+          'type': 'executable',
+          'dependencies': [
+            '../base/base.gyp:base',
+            'mojo_base.gyp:mojo_common_lib',
+            'mojo_base.gyp:mojo_environment_chromium',
+            'mojo_shell_lib',
+          ],
+          'sources': [
+            'shell/desktop/mojo_main.cc',
+          ],
+          'conditions': [
+            ['component=="shared_library"', {
+              'dependencies': [
+                '../ui/gfx/gfx.gyp:gfx',
+              ],
+            }],
+          ],
+        },
+      ],
+    }],
     ['OS=="android"', {
       'targets': [
         {

mojo/public/VERSION

+00f7cdc52b6fcab8deda7c5e266c2db8ba60a28a

mojo/public/mojo.gni

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+declare_args() {
+  # Whether to use a prebuilt mojo_shell binary instead of one built from
+  # source.
+  use_prebuilt_mojo_shell = false
+}

mojo/public/tools/BUILD.gn

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//mojo/public/mojo.gni")
+
+if (use_prebuilt_mojo_shell) {
+  copy("copy_mojo_shell") {
+    filename = "mojo_shell"
+    if (is_win) {
+      filename += ".exe"
+    }
+    sources = [ "prebuilt/$filename" ]
+    outputs = [ "$root_out_dir/$filename" ]
+  }
+}

mojo/public/tools/download_shell_binary.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import subprocess
+import sys
+import tempfile
+import zipfile
+
+current_path = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(current_path, "..", "..", "..", "tools"))
+# pylint: disable=F0401
+import find_depot_tools
+
+if not sys.platform.startswith("linux"):
+  print "Not supported for your platform"
+  sys.exit(0)
+
+version_path = os.path.join(current_path, "../VERSION")
+with open(version_path) as version_file:
+  version = version_file.read().strip()
+
+prebuilt_file_path = os.path.join(current_path, "prebuilt")
+stamp_path = os.path.join(prebuilt_file_path, "VERSION")
+
+try:
+  with open(stamp_path) as stamp_file:
+    current_version = stamp_file.read().strip()
+    if current_version == version:
+      sys.exit(0)
+except IOError:
+  pass
+
+platform = "linux-x64" # TODO: configurate
+basename = platform + ".zip"
+
+gs_path = "gs://mojo/shell/" + version + "/" + basename
+
+depot_tools_path = find_depot_tools.add_depot_tools_to_path()
+gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
+
+with tempfile.NamedTemporaryFile() as temp_zip_file:
+  subprocess.check_call([gsutil_exe, "cp", gs_path, temp_zip_file.name])
+  with zipfile.ZipFile(temp_zip_file.name) as z:
+    zi = z.getinfo("mojo_shell")
+    mode = zi.external_attr >> 16L
+    z.extract(zi, prebuilt_file_path)
+    os.chmod(os.path.join(prebuilt_file_path, "mojo_shell"), mode)
+
+with open(stamp_path, 'w') as stamp_file:
+  stamp_file.write(version + "\n")

mojo/shell/BUILD.gn

@@ -2,9 +2,13 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-import("//mojo/public/tools/bindings/mojom.gni")
 import("//build/config/ui.gni")
+import("//mojo/public/mojo.gni")
+import("//mojo/public/tools/bindings/mojom.gni")
 
+if (!use_prebuilt_mojo_shell) {
+
+# GYP version: mojo/mojo.gyp:mojo_shell
 executable("mojo_shell") {
   deps = [
     ":lib",
@@ -22,6 +26,8 @@ executable("mojo_shell") {
   ]
 }
 
+}  # !use_prebuilt_mojo_shell
+
 # GYP version: mojo/mojo.gyp:mojo_shell_lib
 source_set("lib") {
   deps = [

mojo/tools/upload_shell_binary.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import subprocess
+import sys
+import tempfile
+import time
+import zipfile
+
+root_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                            "..", "..")
+version_path = os.path.join(root_path, "mojo", "public", "VERSION")
+version_file = open(version_path)
+version = version_file.read().strip()
+
+sys.path.insert(0, os.path.join(root_path, "tools"))
+# pylint: disable=F0401
+import find_depot_tools
+
+
+binary_path = os.path.join(root_path, "out", "Release", "mojo_shell")
+
+dest = "gs://mojo/shell/" + version + "/linux-x64.zip"
+
+depot_tools_path = find_depot_tools.add_depot_tools_to_path()
+gsutil_exe = os.path.join(depot_tools_path, "third_party", "gsutil", "gsutil")
+
+with tempfile.NamedTemporaryFile() as zip_file:
+  with zipfile.ZipFile(zip_file, 'w') as z:
+    with open(binary_path) as shell_binary:
+      zipinfo = zipfile.ZipInfo("mojo_shell")
+      zipinfo.external_attr = 0777 << 16L
+      zipinfo.compress_type = zipfile.ZIP_DEFLATED
+      zipinfo.date_time = time.gmtime(os.path.getmtime(binary_path))
+      z.writestr(zipinfo, shell_binary.read())
+  subprocess.check_call([gsutil_exe, "cp", zip_file.name, dest])

tools/find_depot_tools.py

@@ -12,7 +12,7 @@ import sys
 
 
 def IsRealDepotTools(path):
-  return os.path.isfile(os.path.join(path, 'gclient'))
+  return os.path.isfile(os.path.join(path, 'gclient.py'))
 
 
 def add_depot_tools_to_path():