Build naclmono packages based on the sdk manifest.

Re-purpose linux-sdk-mono64 bot to build these release packages, since
all the continuous building is happening on linux-sdk-mono32, this one is free.

BUG=115363
TEST=bots,manual

Note: This CL only affects the Native Client SDK
Review URL: https://chromiumcodereview.appspot.com/10156003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@133731 0039d316-1c4b-4281-b951-d872f2087c98

native_client_sdk/src/build_tools/buildbot_run.py

@@ -24,7 +24,7 @@ SDK_BUILDER_MAP = {
     'linux-sdk-mono32':
         [sys.executable, 'nacl-mono-buildbot.py'],
     'linux-sdk-mono64':
-        ['/bin/true'],
+        [sys.executable, 'nacl-mono-buildbot.py'],
     'DEFAULT':
         [sys.executable, 'build_sdk.py'],
 }

native_client_sdk/src/build_tools/nacl-mono-archive.py

@@ -21,9 +21,9 @@ def main(args):
                     help='Tarfile path',
                     dest='tar_path',
                     default='naclmono_%pepperrev%.bz2')
-  parser.add_option('--sdk-revision',
-                    help='SDK Revision',
-                    dest='sdk_revision',
+  parser.add_option('--upload-path',
+                    help='Upload path (nativeclient-mirror/nacl/nacl_sdk/XXX)',
+                    dest='upload_path',
                     default=None)
   parser.add_option('--pepper-revision',
                     help='Pepper revision',
@@ -35,8 +35,8 @@ def main(args):
                     dest='skip_upload')
   (options, args) = parser.parse_args(args[1:])
 
-  if not options.sdk_revision:
-    buildbot_common.ErrorExit('--sdk-revision is required')
+  if not options.upload_path:
+    buildbot_common.ErrorExit('--upload-path is required')
   if not options.pepper_revision:
     buildbot_common.ErrorExit('--pepper-revision is required')
 
@@ -45,8 +45,6 @@ def main(args):
 
   install_folders = ['bin', 'etc', 'include', 'lib', 'lib32', 'share']
 
-  root_dir = 'mono_pepper_' + options.pepper_revision
-
   buildbot_common.BuildStep('Archive Build')
   tar_file = None
   buildbot_common.RemoveFile(options.tar_path)
@@ -54,14 +52,14 @@ def main(args):
     tar_file = tarfile.open(options.tar_path, mode='w:bz2')
     for subfolder in install_folders:
       tar_file.add(os.path.join(options.install_dir, subfolder),
-                   arcname=os.path.join(root_dir, subfolder))
+                   arcname=subfolder)
   finally:
     if tar_file:
       tar_file.close()
 
   if not options.skip_upload:
     buildbot_common.Archive(os.path.basename(options.tar_path),
-        'nativeclient-mirror/nacl/nacl_sdk/%s' % options.sdk_revision,
+        'nativeclient-mirror/nacl/nacl_sdk/%s' % options.upload_path,
         cwd=os.path.dirname(os.path.abspath(options.tar_path)))
 
 if __name__ == '__main__':

native_client_sdk/src/build_tools/nacl-mono-buildbot.py

@@ -3,33 +3,179 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import hashlib
+import json
 import os
 import sys
 
 import buildbot_common
 import build_utils
 
+GS_MANIFEST_PATH = 'gs://nativeclient-mirror/nacl/nacl_sdk/'
+SDK_MANIFEST = 'naclsdk_manifest2.json'
+MONO_MANIFEST = 'naclmono_manifest.json' 
 
-def main(args):
-  args = args[1:]
-
-  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
-  assert buildbot_revision
-  sdk_revision = buildbot_revision.split(':')[0]
-  pepper_revision = build_utils.ChromeMajorVersion()
-
+def build_and_upload_mono(sdk_revision, pepper_revision, sdk_url, upload_path, args):
   install_dir = 'naclmono'
   buildbot_common.RemoveDir(install_dir)
 
+  revision_opt = ['--sdk-revision', sdk_revision] if sdk_revision else []
+  url_opt = ['--sdk-url', sdk_url] if sdk_url else []
+
   buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
-                      '--arch', 'x86-32', '--install-dir', install_dir] + args)
+                      '--arch', 'x86-32', '--install-dir', install_dir] +
+                      revision_opt + url_opt + args)
   buildbot_common.Run([sys.executable, 'nacl-mono-builder.py',
-                      '--arch', 'x86-64', '--install-dir', install_dir] + args)
+                      '--arch', 'x86-64', '--install-dir', install_dir] +
+                      revision_opt + url_opt + args)
   buildbot_common.Run([sys.executable, 'nacl-mono-archive.py',
-                      '--sdk-revision', sdk_revision,
+                      '--upload-path', upload_path,
                       '--pepper-revision', pepper_revision,
                       '--install-dir', install_dir] + args)
 
+def get_sdk_build_info():
+  '''Returns a list of dictionaries for versions of NaCl Mono to build which are
+     out of date compared to the SDKs available to naclsdk'''
+
+  # Get a copy of the naclsdk manifest file
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      GS_MANIFEST_PATH + SDK_MANIFEST, '.'])
+  manifest_file = open(SDK_MANIFEST, 'r')
+  sdk_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+
+  pepper_infos = []
+  for key, value in sdk_manifest.items():
+    if key == 'bundles':
+      stabilities = ['stable', 'beta', 'dev', 'post_stable']
+      # Pick pepper_* bundles, need pepper_19 or greater to build Mono
+      bundles = filter(lambda b: (b['stability'] in stabilities
+                                  and 'pepper_' in b['name'])
+                                  and b['version'] >= 19, value)
+      for b in bundles:
+        newdict = {}
+        newdict['pepper_revision'] = str(b['version'])
+        linux_arch = filter(lambda u: u['host_os'] == 'linux', b['archives'])
+        newdict['sdk_url'] = linux_arch[0]['url']
+        newdict['sdk_revision'] = b['revision']
+        newdict['stability'] = b['stability']
+        newdict['naclmono_name'] = 'naclmono_' + newdict['pepper_revision']
+        pepper_infos.append(newdict)
+
+  # Get a copy of the naclmono manifest file
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp',
+      GS_MANIFEST_PATH + MONO_MANIFEST, '.'])
+  manifest_file = open(MONO_MANIFEST, 'r')
+  mono_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+  
+  ret = []
+  mono_manifest_dirty = False
+  # Check to see if we need to rebuild mono based on sdk revision
+  for key, value in mono_manifest.items():
+    if key == 'bundles':
+      for info in pepper_infos:
+        bundle = filter(lambda b: b['name'] == info['naclmono_name'], value)
+        if len(bundle) == 0:
+          info['naclmono_rev'] = '1'
+          ret.append(info)
+        else:
+          if info['sdk_revision'] != bundle[0]['sdk_revision']:
+            # This bundle exists in the mono manifest, bump the revision
+            # for the new build we're about to make.
+            info['naclmono_rev'] = str(bundle[0]['revision'] + 1)
+            ret.append(info)
+          elif info['stability'] != bundle[0]['stability']:
+            # If all that happened was the SDK bundle was promoted in stability,
+            # change only that and re-write the manifest
+            mono_manifest_dirty = True
+            bundle[0]['stability'] = info['stability']
+
+  # re-write the manifest here because there are no bundles to build but
+  # the manifest has changed
+  if mono_manifest_dirty and len(ret) == 0:
+    manifest_file = open(MONO_MANIFEST, 'w')
+    manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
+    manifest_file.close()
+    buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
+        MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
+
+  return ret
+
+def update_mono_sdk_json(infos):
+  '''Update the naclmono manifest with the newly built packages'''
+  if len(infos) == 0:
+    return
+
+  manifest_file = open(MONO_MANIFEST, 'r')
+  mono_manifest = json.loads(manifest_file.read())
+  manifest_file.close()
+
+  newbundles = {}
+  for info in infos:
+    bundle = {}
+    bundle['name'] = info['naclmono_name']
+    bundle['description'] = 'Mono for Native Client'
+    bundle['stability'] = info['stability']
+    bundle['recommended'] = 'no'
+    bundle['version'] = 'experimental'
+    archive = {}
+    sha1_hash = hashlib.sha1()
+    f = open(info['naclmono_name'] + '.bz2', 'rb')
+    sha1_hash.update(f.read())
+    archive['size'] = f.tell()
+    f.close()
+    archive['checksum'] = { 'sha1': sha1_hash.hexdigest() }
+    archive['host_os'] = 'all'
+    archive['url'] = ('https://commondatastorage.googleapis.com/'
+                      'nativeclient-mirror/nacl/nacl_sdk/%s/%s/%s.bz2'
+                      % (info['naclmono_name'], info['naclmono_rev'],
+                        info['naclmono_name']))
+    bundle['archives'] = [archive]
+    bundle['revision'] = int(info['naclmono_rev'])
+    bundle['sdk_revision'] = int(info['sdk_revision'])
+
+    # Insert this new bundle into the manifest,
+    # probably overwriting an existing bundle.
+    for key, value in mono_manifest.items():
+      if key == 'bundles':
+        existing = filter(lambda b: b['name'] == info['naclmono_name'], value)
+        if len(existing) > 0:
+          loc = value.index(existing[0])
+          value[loc] = bundle
+        else:
+          value.append(bundle)
+    
+  # Write out the file locally, then upload to its known location.
+  manifest_file = open(MONO_MANIFEST, 'w')
+  manifest_file.write(json.dumps(mono_manifest, sort_keys=False, indent=2))
+  manifest_file.close()
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', '-a', 'public-read',
+      MONO_MANIFEST, GS_MANIFEST_PATH + MONO_MANIFEST])
+  
+
+def main(args):
+  args = args[1:]
+
+  buildbot_revision = os.environ.get('BUILDBOT_REVISION', '')
+  buildername = os.environ.get('BUILDBOT_BUILDERNAME', '')
+
+  if buildername == 'linux-sdk-mono32':
+    assert buildbot_revision
+    sdk_revision = buildbot_revision.split(':')[0]
+    pepper_revision = build_utils.ChromeMajorVersion()
+    build_and_upload_mono(sdk_revision, pepper_revision, None,
+                          'trunk.' + sdk_revision, args)
+  elif buildername == 'linux-sdk-mono64':
+    infos = get_sdk_build_info()
+    for info in infos:
+      # This will put the file in naclmono_19/1/naclmono_19.bz2 for example.
+      upload_path = info['naclmono_name'] + '/' + info['naclmono_rev']
+      build_and_upload_mono(None, info['pepper_revision'], info['sdk_url'],
+          upload_path, args)
+    update_mono_sdk_json(infos)
+    
+
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv))

native_client_sdk/src/build_tools/nacl-mono-builder.py

@@ -20,11 +20,15 @@ def main(args):
                     help='Target architecture',
                     dest='arch',
                     default='x86-32')
-  parser.add_option('--sdk_version',
-                    help='SDK Version (pepper_[X], r[X])'
+  parser.add_option('--sdk-revision',
+                    help='SDK Revision'
                          ' (default=buildbot revision)',
-                    dest='sdk_version',
-                    default='')
+                    dest='sdk_revision',
+                    default=None)
+  parser.add_option('--sdk-url',
+                    help='SDK Download URL',
+                    dest='sdk_url',
+                    default=None)
   parser.add_option('--install-dir',
                     help='Install Directory',
                     dest='install_dir',
@@ -43,34 +47,39 @@ def main(args):
 
   buildbot_common.BuildStep(build_prefix + 'Setup New SDK')
   sdk_dir = None
-  sdk_revision = None
-  if options.sdk_version == '':
-    assert buildbot_revision
-    sdk_revision = buildbot_revision.split(':')[0]
-    url = 'gs://nativeclient-mirror/nacl/nacl_sdk/'\
-          'trunk.%s/naclsdk_linux.bz2' % sdk_revision
-    buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', url, '.'],
-                        cwd=MONO_BUILD_DIR)
-    tar_file = None
-    try:
-      tar_file = tarfile.open(os.path.join(MONO_BUILD_DIR, 'naclsdk_linux.bz2'))
-      pepper_dir = os.path.commonprefix(tar_file.getnames())
-      tar_file.extractall(path=MONO_BUILD_DIR)
-      sdk_dir = os.path.join(MONO_BUILD_DIR, pepper_dir)
-    finally:
-      if tar_file:
-        tar_file.close()
-  else:
-    buildbot_common.ErrorExit('sdk_version not yet supported')
+  sdk_revision = options.sdk_revision
+  sdk_url = options.sdk_url
+  if not sdk_url:
+    if not sdk_revision:
+      assert buildbot_revision
+      sdk_revision = buildbot_revision.split(':')[0]
+    sdk_url = 'gs://nativeclient-mirror/nacl/nacl_sdk/'\
+              'trunk.%s/naclsdk_linux.bz2' % sdk_revision
+
+  sdk_url = sdk_url.replace('https://commondatastorage.googleapis.com/',
+                            'gs://')
+
+  sdk_file = sdk_url.split('/')[-1]
+
+  buildbot_common.Run([buildbot_common.GetGsutil(), 'cp', sdk_url, sdk_file],
+                      cwd=MONO_BUILD_DIR)
+  tar_file = None
+  try:
+    tar_file = tarfile.open(os.path.join(MONO_BUILD_DIR, sdk_file))
+    pepper_dir = os.path.commonprefix(tar_file.getnames())
+    tar_file.extractall(path=MONO_BUILD_DIR)
+    sdk_dir = os.path.join(MONO_BUILD_DIR, pepper_dir)
+  finally:
+    if tar_file:
+      tar_file.close()
 
   assert sdk_dir
-  assert sdk_revision
 
   buildbot_common.BuildStep(build_prefix + 'Checkout Mono')
   # TODO(elijahtaylor): Get git URL from master/trigger to make this
   # more flexible for building from upstream and release branches.
   git_url = 'git://github.com/elijahtaylor/mono.git'
-  git_rev = None
+  git_rev = 'HEAD'
   if buildbot_revision:
     git_rev = buildbot_revision.split(':')[1]
   if not os.path.exists(MONO_DIR):