Reland SDK manifest generation script & SDK rolls.

The issue which prompted reverts on "mac_upload_clang" and
"linux_upload_clang" had to do with incorrect platform-specific
paths to the GN tool.

Summary of reverts:
Reland "Revert "Roll Fuchsia SDK from 98ba6c8440c2 to b9dd8279f122""
This reverts commit 6be9416c.

Reland "Revert "[Fuchsia] Remove namespace_path parameter from FIDL targets""
This reverts commit c354aa9a.

Reland "Revert "[fuchsia] Reland "Generate BUILD.gn from SDK metadata..."""
This reverts commit 712b3456.

TBR=sky@chromium.org

Bug: 889839
Cq-Include-Trybots: luci.chromium.try:android_optional_gpu_tests_rel;luci.chromium.try:linux_optional_gpu_tests_rel;luci.chromium.try:mac_optional_gpu_tests_rel;luci.chromium.try:win_optional_gpu_tests_rel
Change-Id: I46650fb6c7887393c409bd13d9f0851a5630c88e
Reviewed-on: https://chromium-review.googlesource.com/1252013
Commit-Queue: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Kevin Marshall <kmarshall@chromium.org>
Reviewed-by: Scott Graham <scottmg@chromium.org>
Cr-Commit-Position: refs/heads/master@{#595689}

base/BUILD.gn

@@ -2201,7 +2201,6 @@ if (is_ios || is_mac) {
 if (is_fuchsia) {
   fidl_library("testfidl") {
     namespace = "base.fuchsia"
-    namespace_path = "base/fuchsia"
 
     sources = [
       "fuchsia/test.fidl",

build/config/fuchsia/fidl_library.gni

@@ -12,13 +12,6 @@ assert(is_fuchsia)
 #   library_name   - (optional) Name of the library. target_name is used if name
 #                    is not specified explicitly.
 #   namespace      - (optional) Namespace for the library.
-#   namespace_path - (optional) namespace with '.' are replaced with '/', e.g.
-#                    if namespace is "fuchsia.foo", then namespace_path must be
-#                    set to "fuchsia/foo". This parameter must be specified when
-#                    namespace is specified.
-#                    TODO(sergeyu): In theory namespace_path can be generated
-#                    from name, but GN doesn't provide any tools to perform this
-#                    conversion without invoking a python script.
 #   deps           - (optional) List of other fidl_library() targets that this
 #                    FIDL library depends on.
 #
@@ -26,11 +19,7 @@ assert(is_fuchsia)
 # files.
 
 template("fidl_library") {
-  forward_variables_from(invoker,
-                         [
-                           "namespace",
-                           "namespace_path",
-                         ])
+  forward_variables_from(invoker, [ "namespace" ])
 
   _library_basename = target_name
   if (defined(invoker.library_name)) {
@@ -38,10 +27,9 @@ template("fidl_library") {
   }
 
   if (defined(namespace)) {
-    assert(defined(namespace_path),
-           "FIDL libraries with namespace must specify namespace_path")
     _library_name = "${namespace}.${_library_basename}"
-    _library_path = "${namespace_path}/${_library_basename}"
+    _namespace_path = string_replace(namespace, ".", "/")
+    _library_path = "${_namespace_path}/${_library_basename}"
   } else {
     _library_name = _library_basename
     _library_path = _library_basename

build/fuchsia/linux.sdk.sha1

-065fd902445d69829dddf465c524700b375de112
\ No newline at end of file
+065fd902445d69829dddf465c524700b375de112

build/fuchsia/mac.sdk.sha1

-4dfe28e996ad104738f8abdb1f27680b369dba9a
\ No newline at end of file
+4dfe28e996ad104738f8abdb1f27680b369dba9a

build/fuchsia/update_sdk.py

@@ -75,7 +75,8 @@ def main():
   # Previously SDK was unpacked in //third_party/fuchsia-sdk instead of
   # //third_party/fuchsia-sdk/sdk . Remove the old files if they are still
   # there.
-  Cleanup(os.path.join(REPOSITORY_ROOT, 'third_party', 'fuchsia-sdk'))
+  sdk_root = os.path.join(REPOSITORY_ROOT, 'third_party', 'fuchsia-sdk')
+  Cleanup(sdk_root)
 
   hash_file = GetSdkHashForPlatform()
   with open(hash_file, 'r') as f:
@@ -85,14 +86,15 @@ def main():
     print >>sys.stderr, 'No SHA1 found in %s' % hash_file
     return 1
 
-  output_dir = os.path.join(REPOSITORY_ROOT, 'third_party', 'fuchsia-sdk',
-                            'sdk')
+  output_dir = os.path.join(sdk_root, 'sdk')
 
   hash_filename = os.path.join(output_dir, '.hash')
   if os.path.exists(hash_filename):
     with open(hash_filename, 'r') as f:
       if f.read().strip() == sdk_hash:
-        # Nothing to do.
+        # Nothing to do. Generate sdk/BUILD.gn anyways, in case the conversion
+        # script changed.
+        subprocess.check_call([os.path.join(sdk_root, 'gen_build_defs.py')])
         return 0
 
   print 'Downloading SDK %s...' % sdk_hash
@@ -113,6 +115,9 @@ def main():
   finally:
     os.remove(tmp)
 
+  # Generate sdk/BUILD.gn.
+  subprocess.check_call([os.path.join(sdk_root, 'gen_build_defs.py')])
+
   with open(hash_filename, 'w') as f:
     f.write(sdk_hash)
 

chromecast/media/cma/backend/fuchsia/BUILD.gn

@@ -40,7 +40,7 @@ cast_source_set("cma_backend_support") {
     "//chromecast/public",
     "//chromecast/public/media",
     "//media",
-    "//third_party/fuchsia-sdk:media",
+    "//third_party/fuchsia-sdk/sdk:media",
   ]
 }
 
@@ -56,6 +56,6 @@ cast_source_set("unit_tests") {
     ":cma_backend_support",
     "//base",
     "//testing/gtest",
-    "//third_party/fuchsia-sdk:media",
+    "//third_party/fuchsia-sdk/sdk:media",
   ]
 }

content/common/BUILD.gn

@@ -469,8 +469,8 @@ source_set("common") {
 
     deps += [
       "//third_party/fuchsia-sdk:fdio",
-      "//third_party/fuchsia-sdk:fonts",
-      "//third_party/fuchsia-sdk:scenic",
+      "//third_party/fuchsia-sdk/sdk:fonts",
+      "//third_party/fuchsia-sdk/sdk:scenic",
     ]
   }
 }

content/shell/BUILD.gn

@@ -343,7 +343,7 @@ jumbo_static_library("content_shell_lib") {
       "//components/crash/content/app:test_support",
       "//components/crash/content/browser",
     ]
-    deps += [ "//third_party/fuchsia-sdk:policy" ]
+    deps += [ "//third_party/fuchsia-sdk/sdk:policy" ]
   }
 
   # Annoyingly, this target and layouttest_support have circular includes.

media/audio/BUILD.gn

@@ -323,7 +323,7 @@ source_set("audio") {
       "fuchsia/audio_output_stream_fuchsia.cc",
       "fuchsia/audio_output_stream_fuchsia.h",
     ]
-    deps += [ "//third_party/fuchsia-sdk:media" ]
+    deps += [ "//third_party/fuchsia-sdk/sdk:media" ]
   }
 
   configs += [ "//build/config/compiler:no_size_t_to_int_warning" ]

net/BUILD.gn

@@ -2178,7 +2178,7 @@ source_set("net_deps") {
     }
 
     if (is_fuchsia) {
-      public_deps += [ "//third_party/fuchsia-sdk:netstack" ]
+      public_deps += [ "//third_party/fuchsia-sdk/sdk:netstack" ]
     }
 
     if (use_platform_icu_alternatives) {
@@ -5374,7 +5374,7 @@ test("net_unittests") {
     use_test_server = true
     deps += [
       "//third_party/fuchsia-sdk:fidl_cpp",
-      "//third_party/fuchsia-sdk:netstack",
+      "//third_party/fuchsia-sdk/sdk:netstack",
     ]
     sources += [ "base/network_change_notifier_fuchsia_unittest.cc" ]
   }

skia/BUILD.gn

@@ -499,7 +499,7 @@ component("skia") {
       "ext/fontmgr_fuchsia.h",
     ]
     deps += [
-      "//third_party/fuchsia-sdk:fonts",
+      "//third_party/fuchsia-sdk/sdk:fonts",
       "//third_party/icu:icuuc",
     ]
   }
@@ -823,7 +823,7 @@ test("skia_unittests") {
   if (is_fuchsia) {
     sources += [ "ext/fontmgr_fuchsia_unittest.cc" ]
     deps += [
-      "//third_party/fuchsia-sdk:fonts",
+      "//third_party/fuchsia-sdk/sdk:fonts",
       "//third_party/test_fonts",
     ]
     data_deps = [

third_party/fuchsia-sdk/BUILD.gn

@@ -153,170 +153,6 @@ fuchsia_sdk_pkg("fit") {
   ]
 }
 
-fuchsia_sdk_fidl_pkg("fonts") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-  sources = [
-    "font_provider.fidl",
-  ]
-  deps = [
-    ":mem",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("gfx") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-  sources = [
-    "commands.fidl",
-    "display_info.fidl",
-    "events.fidl",
-    "hit.fidl",
-    "nodes.fidl",
-    "renderer.fidl",
-    "resources.fidl",
-    "shapes.fidl",
-    "types.fidl",
-  ]
-  deps = [
-    ":images",
-    ":mem",
-    ":vectorial",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("images") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-  sources = [
-    "encoded_image.fidl",
-    "image_info.fidl",
-    "image_pipe.fidl",
-    "memory_type.fidl",
-    "presentation_info.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("input") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-  sources = [
-    "commands.fidl",
-    "ime_service.fidl",
-    "input_connection.fidl",
-    "input_device_registry.fidl",
-    "input_dispatcher.fidl",
-    "input_event_constants.fidl",
-    "input_events.fidl",
-    "input_reports.fidl",
-    "text_editing.fidl",
-    "text_input.fidl",
-    "usages.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("math") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-  sources = [
-    "math.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("media") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-
-  sources = [
-    "audio.fidl",
-    "audio_capturer.fidl",
-    "audio_device_enumerator.fidl",
-    "audio_renderer.fidl",
-    "gain_control.fidl",
-    "stream.fidl",
-    "stream_type.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("mem") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-
-  sources = [
-    "buffer.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("netstack") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-
-  sources = [
-    "net_address.fidl",
-    "netstack.fidl",
-  ]
-
-  deps = [
-    ":ethernet",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("ethernet") {
-  namespace = "zircon"
-  namespace_path = "zircon"
-
-  sources = [
-    "ethernet.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("oldhttp") {
-  namespace = "fuchsia.net"
-  namespace_path = "fuchsia/net"
-  sources = [
-    "http_error.fidl",
-    "http_header.fidl",
-    "http_service.fidl",
-    "url_body.fidl",
-    "url_loader.fidl",
-    "url_request.fidl",
-    "url_response.fidl",
-  ]
-  deps = [
-    ":mem",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("policy") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-  sources = [
-    "display_usage.fidl",
-    "presentation.fidl",
-    "presenter.fidl",
-  ]
-  deps = [
-    ":viewsv1",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("scenic") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-  sources = [
-    "commands.fidl",
-    "events.fidl",
-    "scenic.fidl",
-    "session.fidl",
-  ]
-  deps = [
-    ":gfx",
-    ":input",
-    ":sys",
-    ":views",
-  ]
-}
-
 fuchsia_sdk_pkg("svc") {
   sources = [
     "include/lib/svc/dir.h",
@@ -325,79 +161,6 @@ fuchsia_sdk_pkg("svc") {
   libs = [ "svc" ]
 }
 
-fuchsia_sdk_fidl_pkg("sys") {
-  namespace = "fuchsia"
-  namespace_path = "fuchsia"
-
-  sources = [
-    "component_controller.fidl",
-    "environment.fidl",
-    "environment_controller.fidl",
-    "flat_namespace.fidl",
-    "launcher.fidl",
-    "loader.fidl",
-    "runner.fidl",
-    "service_provider.fidl",
-  ]
-  deps = [
-    ":mem",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("vectorial") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-  sources = [
-    "commands.fidl",
-    "events.fidl",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("views") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-
-  sources = [
-    "commands.fidl",
-    "events.fidl",
-  ]
-  deps = [
-    ":gfx",
-    ":images",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("viewsv1") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-
-  sources = [
-    "view_containers.fidl",
-    "view_manager.fidl",
-    "view_properties.fidl",
-    "view_provider.fidl",
-    "view_tree_token.fidl",
-    "view_trees.fidl",
-    "views.fidl",
-  ]
-  deps = [
-    ":gfx",
-    ":images",
-    ":math",
-    ":scenic",
-    ":viewsv1token",
-  ]
-}
-
-fuchsia_sdk_fidl_pkg("viewsv1token") {
-  namespace = "fuchsia.ui"
-  namespace_path = "fuchsia/ui"
-
-  sources = [
-    "view_token.fidl",
-  ]
-}
-
 fuchsia_sdk_pkg("zx") {
   sources = [
     "channel.cpp",

third_party/fuchsia-sdk/fuchsia_sdk_pkg.gni

@@ -13,8 +13,7 @@ import("//build/config/fuchsia/fidl_library.gni")
 # Parameters:
 #   package_name - Name of the library. target_name is used if name
 #                  is not specified explicitly.
-#   namespace, namespace_path - FIDL namespace. See fidl_library.gni
-#                               for details.
+#   namespace    - FIDL namespace.
 #   sources      - List of sources relative to sdk/fidl/${name}.
 #   deps         - List of dependencies.
 template("fuchsia_sdk_fidl_pkg") {
@@ -35,20 +34,10 @@ template("fuchsia_sdk_fidl_pkg") {
     library_name = _package_name
 
     if (defined(invoker.namespace)) {
-      assert(defined(invoker.namespace_path),
-             "SDK packages with namespace must specify namespace_path")
       namespace = invoker.namespace
-      namespace_path = invoker.namespace_path
-
-      _library_name = "${namespace}.${_package_name}"
-    } else {
-      _library_name = _package_name
     }
 
-    sources = []
-    foreach(file, invoker.sources) {
-      sources += [ "sdk/fidl/${_library_name}/${file}" ]
-    }
+    sources = invoker.sources
   }
 }
 

third_party/fuchsia-sdk/gen_build_defs.py

+#!/usr/bin/env python
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+#
+# Generates a single BUILD.gn file with build targets generated using the
+# manifest files in the SDK.
+
+import json
+import os
+import subprocess
+import sys
+
+
+# Inserted at the top of the generated BUILD.gn file.
+_GENERATED_PREAMBLE = """# DO NOT EDIT! This file was generated by
+# //third_party/fuchsia-sdk/gen_build_defs.py.
+# Any changes made to this file will be discarded.
+
+import("//third_party/fuchsia-sdk/fuchsia_sdk_pkg.gni")
+
+"""
+
+
+def SerializeListOfStrings(strings):
+  """Outputs a list of strings in GN-friendly, double-quoted format."""
+
+  return '[' + ','.join(['"{}"'.format(s) for s in strings]) + ']'
+
+
+def ConvertFidlLibrary(json):
+  """Massages the manifest data to match the format required by the GN build
+  and outputs a GN target for the FIDL library
+
+  Arguments:
+    json: The parsed manifest JSON.
+  Returns:
+    The GN target definition, represented as a string."""
+
+  json['deps'] = SerializeListOfStrings(
+      [':' + dep.split('.')[-1] for dep in json['deps']])
+  json['sources'] = SerializeListOfStrings(json['sources'])
+
+  name_tokenized = json['name'].split('.')
+  json['shortname'] = name_tokenized[-1]
+  json['namespace'] = '.'.join(name_tokenized[:-1])
+
+  return """fuchsia_sdk_fidl_pkg("{shortname}") {{
+    namespace = "{namespace}"
+    public_deps = {deps}
+    sources = {sources}
+  }}""".format(**json)
+
+
+def ConvertNoOp(json):
+  """Null implementation of a conversion function. No output is generated."""
+
+  return ""
+
+
+"""Maps manifest type strings to relevant conversion functions."""
+_CONVERSION_FUNCTION_MAP = {
+  'fidl_library': ConvertFidlLibrary,
+
+  # TODO(888753): Add conversion routines for these manifest types and migrate
+  # existing dependents of //third_party/fuchsia_sdk/BUILD.gn to use the
+  # generated targets instead. The migration can be executed incrementally.
+  'cc_source_library': ConvertNoOp,
+  'cc_prebuilt_library': ConvertNoOp,
+  'host_tool': ConvertNoOp,
+  'image': ConvertNoOp,
+  'loadable_module': ConvertNoOp,
+  'sysroot': ConvertNoOp,
+}
+
+
+def ConvertSdkManifests():
+  sdk_base_dir = os.path.join(os.path.dirname(__file__), 'sdk')
+  toplevel_meta = json.load(open(os.path.join(sdk_base_dir, 'meta',
+                                              'manifest.json')))
+
+  build_output_path = os.path.join(sdk_base_dir, 'BUILD.gn')
+  with open(build_output_path, 'w') as buildfile:
+    buildfile.write(_GENERATED_PREAMBLE)
+
+    for next_part in toplevel_meta['parts']:
+      parsed = json.load(open(os.path.join(sdk_base_dir, next_part)))
+      if 'type' not in parsed:
+        raise Exception("Couldn't find 'type' node in %s." % next_part)
+
+      convert_function = _CONVERSION_FUNCTION_MAP.get(parsed['type'])
+      if convert_function is None:
+        raise Exception('Unexpected SDK artifact type %s in %s.' %
+                        (parsed['type'], next_part))
+      buildfile.write(convert_function(parsed))
+
+
+if __name__ == '__main__':
+  sys.exit(ConvertSdkManifests())

tools/fuchsia/local-sdk.py

@@ -91,6 +91,10 @@ def main(args):
   shutil.rmtree(tempdir)
   os.chdir(original_dir)
 
+  subprocess.check_call([os.path.join(REPOSITORY_ROOT, 'third_party',
+                                      'fuchsia-sdk',
+                                      'gen_build_defs.py')])
+
   return 0
 
 

ui/base/ime/BUILD.gn

@@ -243,7 +243,7 @@ jumbo_component("ime") {
     ]
 
     deps += [
-      "//third_party/fuchsia-sdk:input",
+      "//third_party/fuchsia-sdk/sdk:input",
       "//ui/events",
       "//ui/events:dom_keycode_converter",
     ]

ui/events/BUILD.gn

@@ -306,7 +306,7 @@ jumbo_component("events") {
       "fuchsia/input_event_dispatcher.h",
       "fuchsia/input_event_dispatcher_delegate.h",
     ]
-    public_deps += [ "//third_party/fuchsia-sdk:input" ]
+    public_deps += [ "//third_party/fuchsia-sdk/sdk:input" ]
   }
 }
 

ui/ozone/demo/BUILD.gn

@@ -39,7 +39,7 @@ source_set("ozone_demo_lib") {
   ]
 
   if (is_fuchsia) {
-    deps += [ "//third_party/fuchsia-sdk:policy" ]
+    deps += [ "//third_party/fuchsia-sdk/sdk:policy" ]
   }
 }
 

ui/ozone/platform/scenic/BUILD.gn

@@ -33,12 +33,12 @@ source_set("scenic") {
   deps = [
     "//base",
     "//skia",
-    "//third_party/fuchsia-sdk:gfx",
-    "//third_party/fuchsia-sdk:images",
-    "//third_party/fuchsia-sdk:mem",
-    "//third_party/fuchsia-sdk:scenic",
-    "//third_party/fuchsia-sdk:viewsv1",
-    "//third_party/fuchsia-sdk:viewsv1token",
+    "//third_party/fuchsia-sdk/sdk:gfx",
+    "//third_party/fuchsia-sdk/sdk:images",
+    "//third_party/fuchsia-sdk/sdk:mem",
+    "//third_party/fuchsia-sdk/sdk:scenic",
+    "//third_party/fuchsia-sdk/sdk:viewsv1",
+    "//third_party/fuchsia-sdk/sdk:viewsv1token",
     "//ui/base",
     "//ui/display/manager",
     "//ui/events:dom_keycode_converter",

ui/platform_window/BUILD.gn

@@ -24,7 +24,7 @@ source_set("platform_window") {
 
   if (is_fuchsia) {
     public_deps = [
-      "//third_party/fuchsia-sdk:viewsv1token",
+      "//third_party/fuchsia-sdk/sdk:viewsv1token",
     ]
   }
 }

webrunner/BUILD.gn

@@ -211,7 +211,6 @@ test("webrunner_unittests") {
 fidl_library("fidl") {
   library_name = "web"
   namespace = "chromium"
-  namespace_path = "chromium"
 
   sources = [
     "fidl/context.fidl",
@@ -222,9 +221,9 @@ fidl_library("fidl") {
   ]
 
   public_deps = [
-    "//third_party/fuchsia-sdk:gfx",
-    "//third_party/fuchsia-sdk:sys",
-    "//third_party/fuchsia-sdk:viewsv1",
+    "//third_party/fuchsia-sdk/sdk:gfx",
+    "//third_party/fuchsia-sdk/sdk:sys",
+    "//third_party/fuchsia-sdk/sdk:viewsv1",
   ]
 }
 

webrunner/net_http/BUILD.gn

@@ -21,8 +21,8 @@ executable("http_service") {
   deps = [
     "//base:base",
     "//net:net",
-    "//third_party/fuchsia-sdk:oldhttp",
-    "//third_party/fuchsia-sdk:sys",
+    "//third_party/fuchsia-sdk/sdk:oldhttp",
+    "//third_party/fuchsia-sdk/sdk:sys",
   ]
 }