Introduce ResourceSnapshotForWebBundle and implement in renderer process

This CL introduces ResourceSnapshotForWebBundle mojo interface and
GetResourceSnapshotForWebBundle() method in LocalFrame mojo interface.
This method will be used to create a web bundle file.
Design Doc: https://docs.google.com/document/d/1O1fH7ev2NwRqOVd_WjEtv4JaI6fA4x6G3ogfz6Hi9Pc/edit#
This is the working-in-progress CL: https://crrev.com/c/1994848
  - The browser process will call GetResourceSnapshotForWebBundle()
    to get the Mojo endpoint of the interface.
  - The browser process will pass the Mojo endpoint to the utility
    (data decoder) process.
  - The utility process will calls the methods of the interface using
    the passed Mojo endpoint, and will generate a Web Bundle file.

Basically this CL implements GetResourceSnapshotForWebBundle() method
using the existing page serialization logic for MHTML generation.
For web bundle generation, we need to change the serialization logic:
  - Subframe URL replacement
  - Style element URL replacement
  - Shadow DOM support
But this CL doesn't implement the web bundle specific logic.
We will create another CL to implement them.

Bug: 1040752
Change-Id: Ia7470de2cd1e418d8bc2d393f5136042145f62d7
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2060360Reviewed-by: Robert Sesek <rsesek@chromium.org>
Reviewed-by: Kinuko Yasuda <kinuko@chromium.org>
Reviewed-by: Kunihiko Sakamoto <ksakamoto@chromium.org>
Commit-Queue: Tsuyoshi Horo <horo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#746265}

content/browser/web_package/save_page_as_web_bundle_browsertest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "base/base_paths.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/path_service.h"
+#include "base/strings/stringprintf.h"
+#include "base/test/bind_test_util.h"
+#include "content/browser/frame_host/render_frame_host_impl.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/test/content_browser_test.h"
+#include "content/public/test/content_browser_test_utils.h"
+#include "content/shell/browser/shell.h"
+#include "services/data_decoder/public/mojom/resource_snapshot_for_web_bundle.mojom.h"
+
+namespace content {
+namespace {
+
+uint64_t GetResourceCount(
+    mojo::Remote<data_decoder::mojom::ResourceSnapshotForWebBundle>& snapshot) {
+  uint64_t count_out = 0;
+  base::RunLoop run_loop;
+  snapshot->GetResourceCount(
+      base::BindLambdaForTesting([&run_loop, &count_out](uint64_t count) {
+        count_out = count;
+        run_loop.Quit();
+      }));
+  run_loop.Run();
+  return count_out;
+}
+
+data_decoder::mojom::SerializedResourceInfoPtr GetResourceInfo(
+    mojo::Remote<data_decoder::mojom::ResourceSnapshotForWebBundle>& snapshot,
+    uint64_t index) {
+  data_decoder::mojom::SerializedResourceInfoPtr info_out;
+  base::RunLoop run_loop;
+  snapshot->GetResourceInfo(
+      index, base::BindLambdaForTesting(
+                 [&run_loop, &info_out](
+                     data_decoder::mojom::SerializedResourceInfoPtr info) {
+                   info_out = std::move(info);
+                   run_loop.Quit();
+                 }));
+  run_loop.Run();
+  return info_out;
+}
+
+base::Optional<mojo_base::BigBuffer> GetResourceBody(
+    mojo::Remote<data_decoder::mojom::ResourceSnapshotForWebBundle>& snapshot,
+    uint64_t index) {
+  base::Optional<mojo_base::BigBuffer> data_out;
+  base::RunLoop run_loop;
+  snapshot->GetResourceBody(
+      index,
+      base::BindLambdaForTesting(
+          [&run_loop, &data_out](base::Optional<mojo_base::BigBuffer> data) {
+            data_out = std::move(data);
+            run_loop.Quit();
+          }));
+  run_loop.Run();
+  return data_out;
+}
+
+}  // namespace
+
+class SavePageAsWebBundleBrowserTest : public ContentBrowserTest {
+ protected:
+  void SetUpOnMainThread() override {
+    embedded_test_server()->AddDefaultHandlers(GetTestDataFilePath());
+    ASSERT_TRUE(embedded_test_server()->Start());
+  }
+
+  mojo::Remote<data_decoder::mojom::ResourceSnapshotForWebBundle>
+  NavigateAndGetSnapshot(const GURL& url) {
+    NavigateToURLBlockUntilNavigationsComplete(shell(), url, 1);
+    mojo::Remote<data_decoder::mojom::ResourceSnapshotForWebBundle> snapshot;
+    static_cast<RenderFrameHostImpl*>(shell()->web_contents()->GetMainFrame())
+        ->GetAssociatedLocalFrame()
+        ->GetResourceSnapshotForWebBundle(
+            snapshot.BindNewPipeAndPassReceiver());
+    return snapshot;
+  }
+};
+
+IN_PROC_BROWSER_TEST_F(SavePageAsWebBundleBrowserTest, OnePageSimple) {
+  const auto page_url = embedded_test_server()->GetURL(
+      "/web_bundle/save_page_as_web_bundle/one_page_simple.html");
+  auto snapshot = NavigateAndGetSnapshot(page_url);
+  ASSERT_EQ(1u, GetResourceCount(snapshot));
+
+  auto info = GetResourceInfo(snapshot, 0);
+  ASSERT_TRUE(info);
+  EXPECT_EQ(page_url, info->url);
+  EXPECT_EQ("text/html", info->mime_type);
+  EXPECT_GT(info->size, 0lu);
+
+  auto data = GetResourceBody(snapshot, 0);
+  ASSERT_TRUE(data);
+  EXPECT_EQ(info->size, data->size());
+
+  EXPECT_EQ(
+      "<html>"
+      "<head>"
+      "<meta http-equiv=\"Content-Type\" content=\"text/html; "
+      "charset=UTF-8\">\n"
+      "<title>Hello</title>\n"
+      "</head>"
+      "<body><h1>hello world</h1>\n</body></html>",
+      std::string(reinterpret_cast<const char*>(data->data()), data->size()));
+
+  // GetResourceInfo() API with an out-of-range index should return null.
+  EXPECT_TRUE(GetResourceInfo(snapshot, 1).is_null());
+  // GetResourceBody() API with an out-of-range index should return nullopt.
+  EXPECT_FALSE(GetResourceBody(snapshot, 1).has_value());
+}
+
+IN_PROC_BROWSER_TEST_F(SavePageAsWebBundleBrowserTest, OnePageWithImg) {
+  const auto page_url = embedded_test_server()->GetURL(
+      "/web_bundle/save_page_as_web_bundle/one_page_with_img.html");
+  const auto img_url = embedded_test_server()->GetURL(
+      "/web_bundle/save_page_as_web_bundle/img.png");
+  auto snapshot = NavigateAndGetSnapshot(page_url);
+  ASSERT_EQ(2u, GetResourceCount(snapshot));
+
+  // The first item of resources must be the page, as FrameSerializer pushes the
+  // SerializedResource of the html content in front of the deque of
+  // SerializedResources.
+  auto page_info = GetResourceInfo(snapshot, 0);
+  ASSERT_TRUE(page_info);
+  EXPECT_EQ(page_url, page_info->url);
+  EXPECT_EQ("text/html", page_info->mime_type);
+  EXPECT_GT(page_info->size, 0lu);
+
+  auto img_info = GetResourceInfo(snapshot, 1u);
+  ASSERT_TRUE(img_info);
+  EXPECT_EQ(img_url, img_info->url);
+  EXPECT_EQ("image/png", img_info->mime_type);
+  EXPECT_GT(img_info->size, 0lu);
+
+  auto page_data = GetResourceBody(snapshot, 0);
+  ASSERT_TRUE(page_data);
+  EXPECT_EQ(page_info->size, page_data->size());
+  EXPECT_EQ(base::StringPrintf(
+                "<html>"
+                "<head>"
+                "<meta http-equiv=\"Content-Type\" content=\"text/html; "
+                "charset=UTF-8\">\n"
+                "<title>Hello</title>\n"
+                "</head>"
+                "<body>"
+                "<img src=\"%s\">\n"
+                "<h1>hello world</h1>\n</body></html>",
+                img_url.spec().c_str()),
+            std::string(reinterpret_cast<const char*>(page_data->data()),
+                        page_data->size()));
+
+  std::string img_file_data;
+  {
+    base::ScopedAllowBlockingForTesting allow_blocking;
+    base::FilePath src_dir;
+    ASSERT_TRUE(base::PathService::Get(base::DIR_SOURCE_ROOT, &src_dir));
+    ASSERT_TRUE(base::ReadFileToString(
+        src_dir.Append(GetTestDataFilePath())
+            .Append(FILE_PATH_LITERAL(
+                "web_bundle/save_page_as_web_bundle/img.png")),
+        &img_file_data));
+  }
+  auto img_data = GetResourceBody(snapshot, 1);
+  EXPECT_EQ(img_file_data,
+            std::string(reinterpret_cast<const char*>(img_data->data()),
+                        img_data->size()));
+}
+
+// TODO(crbug.com/1040752): Implement sub frames support and add tests.
+// TODO(crbug.com/1040752): Implement style sheet support and add tests.
+
+}  // namespace content

content/public/test/fake_local_frame.cc

@@ -50,6 +50,10 @@ void FakeLocalFrame::Focus() {}
 
 void FakeLocalFrame::ClearFocusedElement() {}
 
+void FakeLocalFrame::GetResourceSnapshotForWebBundle(
+    mojo::PendingReceiver<data_decoder::mojom::ResourceSnapshotForWebBundle>
+        receiver) {}
+
 void FakeLocalFrame::CopyImageAt(const gfx::Point& window_point) {}
 
 void FakeLocalFrame::SaveImageAt(const gfx::Point& window_point) {}

content/public/test/fake_local_frame.h

@@ -47,6 +47,9 @@ class FakeLocalFrame : public blink::mojom::LocalFrame {
   void EnableViewSourceMode() override;
   void Focus() override;
   void ClearFocusedElement() override;
+  void GetResourceSnapshotForWebBundle(
+      mojo::PendingReceiver<data_decoder::mojom::ResourceSnapshotForWebBundle>
+          receiver) override;
   void CopyImageAt(const gfx::Point& window_point) override;
   void SaveImageAt(const gfx::Point& window_point) override;
   void ReportBlinkFeatureUsage(

content/test/BUILD.gn

@@ -1036,6 +1036,7 @@ test("content_browsertests") {
     "../browser/web_contents/web_contents_impl_browsertest.cc",
     "../browser/web_contents/web_contents_view_aura_browsertest.cc",
     "../browser/web_contents_receiver_set_browsertest.cc",
+    "../browser/web_package/save_page_as_web_bundle_browsertest.cc",
     "../browser/web_package/signed_exchange_request_handler_browsertest.cc",
     "../browser/web_package/signed_exchange_subresource_prefetch_browsertest.cc",
     "../browser/web_package/web_bundle_browsertest.cc",
@@ -1101,6 +1102,7 @@ test("content_browsertests") {
   public_deps = [
     "//content:content_resources",
     "//content:dev_ui_content_resources",
+    "//services/data_decoder/public/mojom:mojom_resource_snapshot_for_web_bundle",
   ]
 
   deps = [

content/test/data/web_bundle/save_page_as_web_bundle/one_page_simple.html

+<meta charset="utf-8">
+<title>Hello</title>
+<h1>hello world</h1>

content/test/data/web_bundle/save_page_as_web_bundle/one_page_with_img.html

+<meta charset="utf-8">
+<title>Hello</title>
+<img src="./img.png">
+<h1>hello world</h1>

services/data_decoder/public/mojom/BUILD.gn

@@ -25,3 +25,19 @@ mojom("mojom") {
     public_deps += [ "//device/bluetooth/public/mojom" ]
   }
 }
+
+mojom("mojom_resource_snapshot_for_web_bundle") {
+  generate_java = true
+  sources = [ "resource_snapshot_for_web_bundle.mojom" ]
+
+  public_deps = [
+    "//mojo/public/mojom/base",
+    "//url/mojom:url_mojom_gurl",
+  ]
+
+  if (!is_ios) {
+    export_class_attribute_blink = "BLINK_PLATFORM_EXPORT"
+    export_define_blink = "BLINK_PLATFORM_IMPLEMENTATION=1"
+    export_header_blink = "third_party/blink/public/platform/web_common.h"
+  }
+}

services/data_decoder/public/mojom/resource_snapshot_for_web_bundle.mojom

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+module data_decoder.mojom;
+
+import "mojo/public/mojom/base/big_buffer.mojom";
+import "url/mojom/url.mojom";
+
+struct SerializedResourceInfo {
+  url.mojom.Url url;
+  string mime_type;
+  uint64 size;
+};
+
+// An interface to get resource snapshot of a frame to generate a web bundle.
+// This interface is implemented by renderer processes. Its end point is set-up
+// by the browser process and then passed to the utility process.
+interface ResourceSnapshotForWebBundle {
+  // Returns the resource count.
+  GetResourceCount() => (uint64 count);
+  // Returns the resource info at the index.
+  GetResourceInfo(uint64 index) => (SerializedResourceInfo? info);
+  // Returns the body of the resource at the index.
+  GetResourceBody(uint64 index) => (mojo_base.mojom.BigBuffer? data);
+};

third_party/blink/public/mojom/BUILD.gn

@@ -194,6 +194,7 @@ mojom("mojom_platform") {
     "//components/schema_org/common:mojom",
     "//components/services/filesystem/public/mojom",
     "//mojo/public/mojom/base",
+    "//services/data_decoder/public/mojom:mojom_resource_snapshot_for_web_bundle",
     "//services/device/public/mojom",
     "//services/network/public/mojom",
     "//services/network/public/mojom:data_pipe_interfaces",

third_party/blink/public/mojom/frame/frame.mojom

@@ -9,6 +9,7 @@ import "mojo/public/mojom/base/string16.mojom";
 import "mojo/public/mojom/base/unguessable_token.mojom";
 import "mojo/public/mojom/base/text_direction.mojom";
 import "mojo/public/mojom/base/time.mojom";
+import "services/data_decoder/public/mojom/resource_snapshot_for_web_bundle.mojom";
 import "services/network/public/mojom/content_security_policy.mojom";
 import "services/network/public/mojom/fetch_api.mojom";
 import "skia/public/mojom/skcolor.mojom";
@@ -368,6 +369,13 @@ interface LocalFrame {
   // Notifies this frame to clear the focused element (if any).
   ClearFocusedElement();
 
+  // Gets the resource snapshot of the frame for creating Web Bundle.
+  // The instance of ResourceSnapshotForWebBundle in the renderer process will
+  // be kept alive as far as the Mojo endpoint is held and the renderer process
+  // is alive.
+  GetResourceSnapshotForWebBundle(
+      pending_receiver<data_decoder.mojom.ResourceSnapshotForWebBundle> receiver);
+
   // Copies the image at |window_point| to the clipboard (if there indeed is an
   // image at that |window_point|).
   CopyImageAt(gfx.mojom.Point window_point);

third_party/blink/renderer/core/DEPS

@@ -62,6 +62,7 @@ include_rules = [
     "+mojo/public/cpp/bindings",
     "+mojo/public/cpp/system",
     "+mojo/public/mojom/base",
+    "+services/data_decoder/public/mojom/resource_snapshot_for_web_bundle.mojom-blink.h",
     "+services/device/public/mojom/wake_lock.mojom-blink.h",
     "+services/metrics/public",
     "+services/network/public/cpp/cors/cors_error_status.h",

third_party/blink/renderer/core/frame/local_frame.cc

@@ -35,7 +35,9 @@
 #include <utility>
 
 #include "base/metrics/histogram_functions.h"
+#include "mojo/public/cpp/bindings/self_owned_receiver.h"
 #include "mojo/public/cpp/system/message_pipe.h"
+#include "services/data_decoder/public/mojom/resource_snapshot_for_web_bundle.mojom-blink.h"
 #include "services/network/public/cpp/features.h"
 #include "services/network/public/mojom/content_security_policy.mojom-blink.h"
 #include "skia/public/mojom/skcolor.mojom-blink.h"
@@ -88,6 +90,8 @@
 #include "third_party/blink/renderer/core/frame/event_handler_registry.h"
 #include "third_party/blink/renderer/core/frame/frame_console.h"
 #include "third_party/blink/renderer/core/frame/frame_overlay.h"
+#include "third_party/blink/renderer/core/frame/frame_serializer.h"
+#include "third_party/blink/renderer/core/frame/frame_serializer_delegate_impl.h"
 #include "third_party/blink/renderer/core/frame/intervention.h"
 #include "third_party/blink/renderer/core/frame/local_dom_window.h"
 #include "third_party/blink/renderer/core/frame/local_frame_client.h"
@@ -147,6 +151,7 @@
 #include "third_party/blink/renderer/platform/json/json_values.h"
 #include "third_party/blink/renderer/platform/loader/fetch/resource_fetcher.h"
 #include "third_party/blink/renderer/platform/loader/fetch/resource_request.h"
+#include "third_party/blink/renderer/platform/mhtml/serialized_resource.h"
 #include "third_party/blink/renderer/platform/network/network_state_notifier.h"
 #include "third_party/blink/renderer/platform/network/network_utils.h"
 #include "third_party/blink/renderer/platform/runtime_enabled_features.h"
@@ -197,6 +202,72 @@ HitTestResult HitTestResultForRootFramePos(
   return result;
 }
 
+class WebBundleGenerationDelegate
+    : public WebFrameSerializer::MHTMLPartsGenerationDelegate {
+  STACK_ALLOCATED();
+
+ public:
+  WebBundleGenerationDelegate() = default;
+  ~WebBundleGenerationDelegate() = default;
+
+  WebBundleGenerationDelegate(const WebBundleGenerationDelegate&) = delete;
+  WebBundleGenerationDelegate& operator=(const WebBundleGenerationDelegate&) =
+      delete;
+
+  bool ShouldSkipResource(const WebURL& url) override { return false; }
+  bool UseBinaryEncoding() override { return false; }
+  bool RemovePopupOverlay() override { return false; }
+  bool UsePageProblemDetectors() override { return false; }
+};
+
+class ResourceSnapshotForWebBundleImpl
+    : public data_decoder::mojom::blink::ResourceSnapshotForWebBundle {
+ public:
+  explicit ResourceSnapshotForWebBundleImpl(Deque<SerializedResource> resources)
+      : resources_(std::move(resources)) {}
+  ~ResourceSnapshotForWebBundleImpl() override = default;
+
+  ResourceSnapshotForWebBundleImpl(const ResourceSnapshotForWebBundleImpl&) =
+      delete;
+  ResourceSnapshotForWebBundleImpl& operator=(
+      const ResourceSnapshotForWebBundleImpl&) = delete;
+
+  // data_decoder::mojom::blink::ResourceSnapshotForWebBundle:
+  void GetResourceCount(GetResourceCountCallback callback) override {
+    std::move(callback).Run(resources_.size());
+  }
+  void GetResourceInfo(uint64_t index,
+                       GetResourceInfoCallback callback) override {
+    if (index >= resources_.size()) {
+      std::move(callback).Run(nullptr);
+      return;
+    }
+    const auto& resource = resources_.at(SafeCast<WTF::wtf_size_t>(index));
+    auto info = data_decoder::mojom::blink::SerializedResourceInfo::New();
+    info->url = resource.url;
+    info->mime_type = resource.mime_type;
+    info->size = resource.data ? resource.data->size() : 0;
+    std::move(callback).Run(std::move(info));
+  }
+  void GetResourceBody(uint64_t index,
+                       GetResourceBodyCallback callback) override {
+    if (index >= resources_.size()) {
+      std::move(callback).Run(base::nullopt);
+      return;
+    }
+    const auto& resource = resources_.at(SafeCast<WTF::wtf_size_t>(index));
+    if (!resource.data) {
+      std::move(callback).Run(base::nullopt);
+      return;
+    }
+    std::move(callback).Run(
+        mojo_base::BigBuffer(resource.data->CopyAs<std::vector<uint8_t>>()));
+  }
+
+ private:
+  const Deque<SerializedResource> resources_;
+};
+
 }  // namespace
 
 template class CORE_TEMPLATE_EXPORT Supplement<LocalFrame>;
@@ -2156,6 +2227,23 @@ void LocalFrame::ClearFocusedElement() {
     Selection().Clear();
 }
 
+void LocalFrame::GetResourceSnapshotForWebBundle(
+    mojo::PendingReceiver<
+        data_decoder::mojom::blink::ResourceSnapshotForWebBundle> receiver) {
+  Deque<SerializedResource> resources;
+
+  HeapHashSet<WeakMember<const Element>> shadow_template_elements;
+  WebBundleGenerationDelegate web_delegate;
+  FrameSerializerDelegateImpl core_delegate(web_delegate,
+                                            shadow_template_elements);
+  FrameSerializer serializer(resources, core_delegate);
+  serializer.SerializeFrame(*this);
+
+  mojo::MakeSelfOwnedReceiver(
+      std::make_unique<ResourceSnapshotForWebBundleImpl>(std::move(resources)),
+      std::move(receiver));
+}
+
 void LocalFrame::CopyImageAtViewportPoint(const IntPoint& viewport_point) {
   HitTestResult result = HitTestResultForVisualViewportPos(viewport_point);
   if (!IsA<HTMLCanvasElement>(result.InnerNodeOrImageMapImage()) &&

third_party/blink/renderer/core/frame/local_frame.h

@@ -505,6 +505,10 @@ class CORE_EXPORT LocalFrame final : public Frame,
   void EnableViewSourceMode() final;
   void Focus() final;
   void ClearFocusedElement() final;
+  void GetResourceSnapshotForWebBundle(
+      mojo::PendingReceiver<
+          data_decoder::mojom::blink::ResourceSnapshotForWebBundle> receiver)
+      final;
   void CopyImageAt(const gfx::Point& window_point) final;
   void SaveImageAt(const gfx::Point& window_point) final;
   void ReportBlinkFeatureUsage(const Vector<mojom::blink::WebFeature>&) final;