Reland "[Layered API] Implement Layered API infrastructure behind a flag"

This is a reland of bde742c2

This reland contains fixes for:
- Issue 839236 (by adding a dependency in core/script/BUILD.gn), and
- A unit test failure that caused revert
  https://chromium-review.googlesource.com/1042065
  (by adding skip_minify flag as the minifier inserted
  unexpected newlines to scripts)

TBR=kinuko@chromium.org,kouhei@chromium.org

Original change's description:
> [Layered API] Implement Layered API infrastructure behind a flag
>
> This CL
> 1. Maps std:x|y URLs to std:x (according to the draft spec [1]),
> 2. Then maps to std-internal://x/index.js (not spec'ed [2])
>    in GetInternalURL(),
> 3. Then retrieves and serves the script contents for std-internal:
>    URLs in GetSourceText() using ui::ResourceBundle()
>    from blink_resources.pak.
>
> The source JavaScript files for Layered APIs are placed under
> third_party/blink/renderer/core/script/resources/layered_api
> and bundled into blink_resources.pak.
>
> This CL introduces std: and std-internal: schemes that are used in Blink side
> (std: is user-facing while std-internal: is mostly invisible for users)
> but not outside Blink.
>
> [1] Spec:
> https://github.com/drufball/layered-apis/blob/master/spec.md
> [2] Implementation design doc:
> https://docs.google.com/document/d/1V-WaCZQbBcQJRSYSYBb8Y6p0DOdDpiNDSmD41ui_73s/edit
>
> Bug: 829084
> Change-Id: I61fc0a2f17d937db6e859303ab8a84e4b2b87850
> Reviewed-on: https://chromium-review.googlesource.com/1013321
> Commit-Queue: Hiroshige Hayashizaki <hiroshige@chromium.org>
> Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
> Reviewed-by: Kouhei Ueno <kouhei@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#555524}

Bug: 829084, 839236
Change-Id: I37ed111cf924b52f5998c1e6a2b6dd7d1965c813
Reviewed-on: https://chromium-review.googlesource.com/1042616
Commit-Queue: Hiroshige Hayashizaki <hiroshige@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#555961}

third_party/blink/public/blink_resources.grd

@@ -47,6 +47,10 @@
         <include name="IDR_LIST_PICKER_JS" file="../renderer/core/html/forms/resources/listPicker.js" type="BINDATA" compress="gzip"/>
       </if>
       <include name="IDR_AUDIO_SPATIALIZATION_COMPOSITE" file="../renderer/platform/audio/resources/Composite.flac" type="BINDATA"/>
+
+      <!-- Layered API scripts. Should be consistent with kLayeredAPIResources
+           in renderer/core/script/layered_api.cc -->
+      <include name="IDR_LAYERED_API_BLANK_INDEX_JS" file="../renderer/core/script/resources/layered_api/blank/index.js" type="BINDATA" skip_minify="true"/>
     </includes>
   </release>
 </grit>

third_party/blink/renderer/core/BUILD.gn

@@ -2040,6 +2040,7 @@ jumbo_source_set("unit_tests") {
     "scheduler/throttling_test.cc",
     "scheduler/virtual_time_test.cc",
     "script/dynamic_module_resolver_test.cc",
+    "script/layered_api_test.cc",
     "script/mock_script_element_base.h",
     "script/modulator_test.cc",
     "script/module_map_test.cc",

third_party/blink/renderer/core/loader/modulescript/document_module_script_fetcher.cc

@@ -7,9 +7,11 @@
 #include "third_party/blink/renderer/core/execution_context/execution_context.h"
 #include "third_party/blink/renderer/core/inspector/console_message.h"
 #include "third_party/blink/renderer/core/loader/subresource_integrity_helper.h"
+#include "third_party/blink/renderer/core/script/layered_api.h"
 #include "third_party/blink/renderer/platform/loader/cors/cors.h"
 #include "third_party/blink/renderer/platform/loader/fetch/fetch_utils.h"
 #include "third_party/blink/renderer/platform/network/mime/mime_type_registry.h"
+#include "third_party/blink/renderer/platform/runtime_enabled_features.h"
 
 namespace blink {
 
@@ -76,9 +78,42 @@ DocumentModuleScriptFetcher::DocumentModuleScriptFetcher(
 void DocumentModuleScriptFetcher::Fetch(FetchParameters& fetch_params,
                                         ModuleScriptFetcher::Client* client) {
   SetClient(client);
+
+  if (FetchIfLayeredAPI(fetch_params))
+    return;
+
   ScriptResource::Fetch(fetch_params, fetcher_, this);
 }
 
+bool DocumentModuleScriptFetcher::FetchIfLayeredAPI(
+    FetchParameters& fetch_params) {
+  if (!RuntimeEnabledFeatures::LayeredAPIEnabled())
+    return false;
+
+  KURL layered_api_url = blink::layered_api::GetInternalURL(fetch_params.Url());
+
+  if (layered_api_url.IsNull())
+    return false;
+
+  const String source_text = blink::layered_api::GetSourceText(layered_api_url);
+
+  if (source_text.IsNull()) {
+    HeapVector<Member<ConsoleMessage>> error_messages;
+    error_messages.push_back(ConsoleMessage::CreateForRequest(
+        kJSMessageSource, kErrorMessageLevel, "Unexpected data error",
+        fetch_params.Url().GetString(), nullptr, 0));
+    Finalize(base::nullopt, error_messages);
+    return true;
+  }
+
+  ModuleScriptCreationParams params(
+      layered_api_url, source_text,
+      fetch_params.GetResourceRequest().GetFetchCredentialsMode(),
+      kSharableCrossOrigin);
+  Finalize(params, HeapVector<Member<ConsoleMessage>>());
+  return true;
+}
+
 void DocumentModuleScriptFetcher::NotifyFinished(Resource* resource) {
   ClearResource();
 

third_party/blink/renderer/core/loader/modulescript/document_module_script_fetcher.h

@@ -44,6 +44,9 @@ class CORE_EXPORT DocumentModuleScriptFetcher : public ModuleScriptFetcher,
   void Finalize(const base::Optional<ModuleScriptCreationParams>&,
                 const HeapVector<Member<ConsoleMessage>>& error_messages);
 
+  // Returns true if loaded as Layered API.
+  bool FetchIfLayeredAPI(FetchParameters&);
+
   Member<ResourceFetcher> fetcher_;
 };
 

third_party/blink/renderer/core/loader/modulescript/module_tree_linker.cc

@@ -7,9 +7,11 @@
 #include "third_party/blink/renderer/bindings/core/v8/script_module.h"
 #include "third_party/blink/renderer/core/loader/modulescript/module_script_fetch_request.h"
 #include "third_party/blink/renderer/core/loader/modulescript/module_tree_linker_registry.h"
+#include "third_party/blink/renderer/core/script/layered_api.h"
 #include "third_party/blink/renderer/core/script/module_script.h"
 #include "third_party/blink/renderer/platform/bindings/v8_throw_exception.h"
 #include "third_party/blink/renderer/platform/loader/fetch/resource_loading_log.h"
+#include "third_party/blink/renderer/platform/runtime_enabled_features.h"
 #include "third_party/blink/renderer/platform/wtf/vector.h"
 #include "v8/include/v8.h"
 
@@ -128,15 +130,38 @@ void ModuleTreeLinker::AdvanceState(State new_state) {
 }
 
 // https://html.spec.whatwg.org/multipage/webappapis.html#fetch-a-module-script-tree
-void ModuleTreeLinker::FetchRoot(const KURL& url,
+void ModuleTreeLinker::FetchRoot(const KURL& original_url,
                                  const ScriptFetchOptions& options) {
 #if DCHECK_IS_ON()
-  url_ = url;
+  original_url_ = original_url;
   root_is_inline_ = false;
 #endif
 
   AdvanceState(State::kFetchingSelf);
 
+  KURL url = original_url;
+  // <spec
+  // href="https://github.com/drufball/layered-apis/blob/master/spec.md#fetch-a-module-script-graph"
+  // step="1">Set url to the layered API fetching URL for url.</spec>
+  if (RuntimeEnabledFeatures::LayeredAPIEnabled())
+    url = blink::layered_api::ResolveFetchingURL(url);
+
+#if DCHECK_IS_ON()
+  url_ = url;
+#endif
+
+  // <spec
+  // href="https://github.com/drufball/layered-apis/blob/master/spec.md#fetch-a-module-script-graph"
+  // step="2">If url is failure, asynchronously complete this algorithm with
+  // null.</spec>
+  if (!url.IsValid()) {
+    result_ = nullptr;
+    modulator_->TaskRunner()->PostTask(
+        FROM_HERE, WTF::Bind(&ModuleTreeLinker::AdvanceState,
+                             WrapPersistent(this), State::kFinished));
+    return;
+  }
+
   // Step 1. Let visited set be << url >>.
   visited_set_.insert(url);
 
@@ -154,7 +179,8 @@ void ModuleTreeLinker::FetchRootInline(ModuleScript* module_script) {
   // Top-level entry point for [FDaI] for an inline module script.
   DCHECK(module_script);
 #if DCHECK_IS_ON()
-  url_ = module_script->BaseURL();
+  original_url_ = module_script->BaseURL();
+  url_ = original_url_;
   root_is_inline_ = true;
 #endif
 
@@ -487,6 +513,7 @@ ScriptValue ModuleTreeLinker::FindFirstParseError(
 #if DCHECK_IS_ON()
 std::ostream& operator<<(std::ostream& stream, const ModuleTreeLinker& linker) {
   stream << "ModuleTreeLinker[" << &linker
+         << ", original_url=" << linker.original_url_.GetString()
          << ", url=" << linker.url_.GetString()
          << ", inline=" << linker.root_is_inline_ << "]";
   return stream;

third_party/blink/renderer/core/loader/modulescript/module_tree_linker.h

@@ -113,6 +113,7 @@ class CORE_EXPORT ModuleTreeLinker final : public SingleModuleClient {
   size_t num_incomplete_fetches_ = 0;
 
 #if DCHECK_IS_ON()
+  KURL original_url_;
   KURL url_;
   bool root_is_inline_;
 

third_party/blink/renderer/core/script/BUILD.gn

@@ -22,6 +22,8 @@ blink_core_sources("script") {
     "html_parser_script_runner.h",
     "html_parser_script_runner_host.h",
     "ignore_destructive_write_count_incrementer.h",
+    "layered_api.cc",
+    "layered_api.h",
     "modulator.cc",
     "modulator.h",
     "modulator_impl_base.cc",
@@ -54,5 +56,9 @@ blink_core_sources("script") {
     "xml_parser_script_runner_host.h",
   ]
 
+  deps = [
+    "//third_party/blink/public:resources",
+  ]
+
   jumbo_excluded_sources = [ "modulator.cc" ]  # https://crbug.com/716395
 }

third_party/blink/renderer/core/script/layered_api.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/core/script/layered_api.h"
+
+#include "base/stl_util.h"
+#include "third_party/blink/public/resources/grit/blink_resources.h"
+#include "third_party/blink/renderer/platform/media/resource_bundle_helper.h"
+#include "third_party/blink/renderer/platform/wtf/text/string_builder.h"
+
+namespace blink {
+
+namespace layered_api {
+
+namespace {
+
+static const char kStdScheme[] = "std";
+static const char kInternalScheme[] = "std-internal";
+
+struct LayeredAPIResource {
+  const char* path;
+  int resource_id;
+};
+
+const LayeredAPIResource kLayeredAPIResources[] = {
+    {"blank/index.js", IDR_LAYERED_API_BLANK_INDEX_JS},
+};
+
+int GetResourceIDFromPath(const String& path) {
+  for (size_t i = 0; i < base::size(kLayeredAPIResources); ++i) {
+    if (path == kLayeredAPIResources[i].path) {
+      return kLayeredAPIResources[i].resource_id;
+    }
+  }
+  return -1;
+}
+
+bool IsImplemented(const String& name) {
+  return GetResourceIDFromPath(name + "/index.js") >= 0;
+}
+
+}  // namespace
+
+// https://github.com/drufball/layered-apis/blob/master/spec.md#user-content-layered-api-fetching-url
+KURL ResolveFetchingURL(const KURL& url) {
+  // <spec step="1">If url's scheme is not "std", return url.</spec>
+  if (!url.ProtocolIs(kStdScheme))
+    return url;
+
+  // <spec step="2">Let path be url's path[0].</spec>
+  const String path = url.GetPath();
+
+  // <spec step="3">Let identifier be the portion of path before the first
+  // U+007C (|), or all of path if no U+007C is present.</spec>
+  //
+  // <spec step="4">Let fallback be the portion of path after the first U+007C,
+  // or null if no U+007C is present.</spec>
+  String identifier;
+  String fallback;
+  const size_t separator_position = path.find('|');
+  if (separator_position != WTF::kNotFound) {
+    identifier = path.Substring(0, separator_position);
+    fallback = path.Substring(separator_position + 1);
+  } else {
+    identifier = path;
+  }
+
+  // <spec step="5">If the layered API identified by path is implemented by this
+  // user agent, return the result of parsing the concatenation of "std:" with
+  // identifier.</spec>
+  if (IsImplemented(identifier)) {
+    StringBuilder url_string;
+    url_string.Append(kStdScheme);
+    url_string.Append(":");
+    url_string.Append(identifier);
+    return KURL(NullURL(), url_string.ToString());
+  }
+
+  // <spec step="6">If fallback is null, return failure.</spec>
+  if (fallback.IsNull())
+    return NullURL();
+
+  // <spec step="7">Return the result of parsing fallback.</spec>
+  return KURL(NullURL(), fallback);
+}
+
+KURL GetInternalURL(const KURL& url) {
+  if (url.ProtocolIs(kStdScheme)) {
+    StringBuilder url_string;
+    url_string.Append(kInternalScheme);
+    url_string.Append("://");
+    url_string.Append(url.GetPath());
+    url_string.Append("/index.js");
+    return KURL(NullURL(), url_string.ToString());
+  }
+
+  if (url.ProtocolIs(kInternalScheme)) {
+    return url;
+  }
+
+  return NullURL();
+}
+
+String GetSourceText(const KURL& url) {
+  if (!url.ProtocolIs(kInternalScheme))
+    return String();
+
+  String path = url.GetPath();
+  // According to the URL spec, the host/path of "std-internal://foo/bar"
+  // is "foo" and "/bar", respectively, but in Blink they are "" and
+  // "//foo/bar". This is a workaround to get "foo/bar" here.
+  if (path.StartsWith("//")) {
+    path = path.Substring(2);
+  }
+
+  int resource_id = GetResourceIDFromPath(path);
+  if (resource_id < 0)
+    return String();
+
+  return ResourceBundleHelper::GetResourceAsString(resource_id);
+}
+
+}  // namespace layered_api
+
+}  // namespace blink

third_party/blink/renderer/core/script/layered_api.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef THIRD_PARTY_BLINK_RENDERER_CORE_SCRIPT_LAYERED_API_H_
+#define THIRD_PARTY_BLINK_RENDERER_CORE_SCRIPT_LAYERED_API_H_
+
+#include "third_party/blink/renderer/core/core_export.h"
+#include "third_party/blink/renderer/platform/weborigin/kurl.h"
+#include "third_party/blink/renderer/platform/wtf/text/wtf_string.h"
+
+namespace blink {
+
+// Implements Layered API.
+// Spec: https://github.com/drufball/layered-apis/blob/master/spec.md
+// Implementation Design Doc:
+// https://docs.google.com/document/d/1V-WaCZQbBcQJRSYSYBb8Y6p0DOdDpiNDSmD41ui_73s/edit
+namespace layered_api {
+
+// <spec
+// href="https://github.com/drufball/layered-apis/blob/master/spec.md#user-content-layered-api-fetching-url">
+// This operation maps URLs of the form std:x|y to either std:x or y
+// URLs.</spec>
+CORE_EXPORT KURL ResolveFetchingURL(const KURL&);
+
+// Returns std-internal://x/index.js if the URL is Layered API, or null URL
+// otherwise (not specced).
+CORE_EXPORT KURL GetInternalURL(const KURL&);
+
+// Gets source text for std-internal://x/index.js.
+CORE_EXPORT String GetSourceText(const KURL&);
+
+}  // namespace layered_api
+
+}  // namespace blink
+
+#endif

third_party/blink/renderer/core/script/layered_api_test.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "third_party/blink/renderer/core/script/layered_api.h"
+
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace blink {
+
+namespace layered_api {
+
+namespace {
+
+TEST(LayeredAPITest, ResolveFetchingURL) {
+  EXPECT_EQ(ResolveFetchingURL(KURL("https://example.com/")),
+            KURL("https://example.com/"));
+  EXPECT_EQ(ResolveFetchingURL(KURL("std:blank")), KURL("std:blank"));
+  EXPECT_EQ(ResolveFetchingURL(KURL("std:blank|https://fallback.example.com/")),
+            KURL("std:blank"));
+  EXPECT_EQ(ResolveFetchingURL(KURL("std:blank|https://:invalid-url")),
+            KURL("std:blank"));
+  EXPECT_EQ(ResolveFetchingURL(KURL("std:none")), NullURL());
+  EXPECT_EQ(ResolveFetchingURL(KURL("std:none|https://fallback.example.com/")),
+            KURL("https://fallback.example.com/"));
+  EXPECT_FALSE(
+      ResolveFetchingURL(KURL("std:none|https://:invalid-url")).IsValid());
+}
+
+TEST(LayeredAPITest, GetInternalURL) {
+  EXPECT_EQ(GetInternalURL(KURL("https://example.com/")), NullURL());
+
+  EXPECT_EQ(GetInternalURL(KURL("std:blank")),
+            KURL("std-internal://blank/index.js"));
+
+  EXPECT_EQ(GetInternalURL(KURL("std-internal://blank/index.js")),
+            KURL("std-internal://blank/index.js"));
+  EXPECT_EQ(GetInternalURL(KURL("std-internal://blank/foo/bar.js")),
+            KURL("std-internal://blank/foo/bar.js"));
+}
+
+TEST(LayeredAPITest, InternalURLRelativeResolution) {
+  EXPECT_EQ(KURL(KURL("std-internal://blank/index.js"), "./sub.js"),
+            KURL("std-internal://blank/sub.js"));
+  EXPECT_EQ(KURL(KURL("std-internal://blank/index.js"), "/sub.js"),
+            KURL("std-internal://blank/sub.js"));
+  EXPECT_EQ(KURL(KURL("std-internal://blank/index.js"), "./foo/bar.js"),
+            KURL("std-internal://blank/foo/bar.js"));
+  EXPECT_EQ(KURL(KURL("std-internal://blank/foo/bar.js"), "../baz.js"),
+            KURL("std-internal://blank/baz.js"));
+}
+
+TEST(LayeredAPITest, GetSourceText) {
+  EXPECT_EQ(GetSourceText(KURL("std-internal://blank/index.js")), String(""));
+
+  EXPECT_EQ(GetSourceText(KURL("std-internal://blank/not-found.js")), String());
+  EXPECT_EQ(GetSourceText(KURL("std-internal://none/index.js")), String());
+
+  EXPECT_EQ(GetSourceText(KURL("https://example.com/")), String());
+}
+
+}  // namespace
+
+}  // namespace layered_api
+
+}  // namespace blink

third_party/blink/renderer/core/script/modulator.cc

@@ -9,6 +9,7 @@
 #include "third_party/blink/renderer/core/frame/local_dom_window.h"
 #include "third_party/blink/renderer/core/frame/local_frame.h"
 #include "third_party/blink/renderer/core/script/document_modulator_impl.h"
+#include "third_party/blink/renderer/core/script/layered_api.h"
 #include "third_party/blink/renderer/core/script/worker_modulator_impl.h"
 #include "third_party/blink/renderer/core/script/worklet_modulator_impl.h"
 #include "third_party/blink/renderer/core/workers/worklet_global_scope.h"
@@ -86,8 +87,17 @@ KURL Modulator::ResolveModuleSpecifier(const String& module_request,
   // <spec step="1">Apply the URL parser to specifier. If the result is not
   // failure, return the result.</spec>
   KURL url(NullURL(), module_request);
-  if (url.IsValid())
+  if (url.IsValid()) {
+    // <spec
+    // href="https://github.com/drufball/layered-apis/blob/master/spec.md#resolve-a-module-specifier"
+    // step="1">Let parsed be the result of applying the URL parser to
+    // specifier. If parsed is not failure, then return the layered API fetching
+    // URL for parsed.</spec>
+    if (RuntimeEnabledFeatures::LayeredAPIEnabled())
+      return blink::layered_api::ResolveFetchingURL(url);
+
     return url;
+  }
 
   // <spec step="2">If specifier does not start with the character U+002F
   // SOLIDUS (/), the two-character sequence U+002E FULL STOP, U+002F SOLIDUS