[inspector] introduce MHTML capture over protocol

Adds new experimental protocol method, which uses
`WebContents::GenerateMHTML` underneath.

Bug: 901933
Change-Id: I52e707f89378fe3a1488a3cd8afd7cdded1b77ea
Reviewed-on: https://chromium-review.googlesource.com/c/1297732
Commit-Queue: Erik Luo <luoe@chromium.org>
Reviewed-by: Dmitry Gozman <dgozman@chromium.org>
Cr-Commit-Position: refs/heads/master@{#606256}

content/browser/BUILD.gn

@@ -670,6 +670,8 @@ jumbo_source_set("browser") {
     "devtools/protocol/devtools_download_manager_delegate.h",
     "devtools/protocol/devtools_download_manager_helper.cc",
     "devtools/protocol/devtools_download_manager_helper.h",
+    "devtools/protocol/devtools_mhtml_helper.cc",
+    "devtools/protocol/devtools_mhtml_helper.h",
     "devtools/protocol/dom_handler.cc",
     "devtools/protocol/dom_handler.h",
     "devtools/protocol/emulation_handler.cc",

content/browser/devtools/protocol/devtools_mhtml_helper.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "content/browser/devtools/protocol/devtools_mhtml_helper.h"
+
+#include "base/bind.h"
+#include "base/task/post_task.h"
+#include "content/browser/web_contents/web_contents_impl.h"
+#include "content/public/browser/browser_task_traits.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/common/mhtml_generation_params.h"
+#include "storage/browser/blob/shareable_file_reference.h"
+
+namespace content {
+namespace protocol {
+
+namespace {
+
+constexpr base::TaskTraits kBlockingSkippableTraits = {
+    // Requires IO.
+    base::MayBlock(),
+
+    // TaskShutdownBehavior: use SKIP_ON_SHUTDOWN so that the helper's
+    // fields do not suddenly become invalid.
+    base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN};
+
+void ClearFileReferenceOnIOThread(
+    scoped_refptr<storage::ShareableFileReference>) {}
+
+}  // namespace
+
+DevToolsMHTMLHelper::DevToolsMHTMLHelper(
+    base::WeakPtr<PageHandler> page_handler,
+    std::unique_ptr<PageHandler::CaptureSnapshotCallback> callback)
+    : page_handler_(page_handler), callback_(std::move(callback)) {}
+
+DevToolsMHTMLHelper::~DevToolsMHTMLHelper() {
+  if (mhtml_file_.get()) {
+    base::PostTaskWithTraits(
+        FROM_HERE, {BrowserThread::IO},
+        base::BindOnce(&ClearFileReferenceOnIOThread, std::move(mhtml_file_)));
+  }
+}
+
+// static
+void DevToolsMHTMLHelper::Capture(
+    base::WeakPtr<PageHandler> page_handler,
+    std::unique_ptr<PageHandler::CaptureSnapshotCallback> callback) {
+  scoped_refptr<DevToolsMHTMLHelper> helper =
+      new DevToolsMHTMLHelper(page_handler, std::move(callback));
+  base::PostTaskWithTraits(
+      FROM_HERE, kBlockingSkippableTraits,
+      base::BindOnce(&DevToolsMHTMLHelper::CreateTemporaryFile, helper));
+}
+
+void DevToolsMHTMLHelper::CreateTemporaryFile() {
+  if (!base::CreateTemporaryFile(&mhtml_snapshot_path_)) {
+    ReportFailure("Unable to create temporary file");
+    return;
+  }
+  base::PostTaskWithTraits(
+      FROM_HERE, {BrowserThread::IO},
+      base::BindOnce(&DevToolsMHTMLHelper::TemporaryFileCreatedOnIO, this));
+}
+
+void DevToolsMHTMLHelper::TemporaryFileCreatedOnIO() {
+  DCHECK_CURRENTLY_ON(BrowserThread::IO);
+
+  // Setup a ShareableFileReference so the temporary file gets deleted
+  // once it is no longer used.
+  mhtml_file_ = storage::ShareableFileReference::GetOrCreate(
+      mhtml_snapshot_path_,
+      storage::ShareableFileReference::DELETE_ON_FINAL_RELEASE,
+      base::CreateSequencedTaskRunnerWithTraits(
+          {// Requires IO.
+           base::MayBlock(),
+
+           // Because we are using DELETE_ON_FINAL_RELEASE here, the
+           // storage::ScopedFile inside ShareableFileReference requires
+           // a shutdown blocking task runner to ensure that its deletion
+           // task runs.
+           base::TaskShutdownBehavior::BLOCK_SHUTDOWN})
+          .get());
+
+  base::PostTaskWithTraits(
+      FROM_HERE, {BrowserThread::UI},
+      base::BindOnce(&DevToolsMHTMLHelper::TemporaryFileCreatedOnUI, this));
+}
+
+void DevToolsMHTMLHelper::TemporaryFileCreatedOnUI() {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  if (!page_handler_) {
+    ReportFailure("");
+    return;
+  }
+  WebContentsImpl* web_contents = page_handler_->GetWebContents();
+  if (!web_contents) {
+    ReportFailure("No web contents");
+    return;
+  }
+  web_contents->GenerateMHTML(
+      content::MHTMLGenerationParams(mhtml_snapshot_path_),
+      base::BindOnce(&DevToolsMHTMLHelper::MHTMLGeneratedOnUI, this));
+}
+
+void DevToolsMHTMLHelper::MHTMLGeneratedOnUI(int64_t mhtml_file_size) {
+  DCHECK_CURRENTLY_ON(BrowserThread::UI);
+  if (mhtml_file_size <= 0 ||
+      mhtml_file_size > std::numeric_limits<int>::max()) {
+    ReportFailure("Failed to generate MHTML");
+    return;
+  }
+  base::PostTaskWithTraits(
+      FROM_HERE, kBlockingSkippableTraits,
+      base::BindOnce(&DevToolsMHTMLHelper::ReadMHTML, this));
+}
+
+void DevToolsMHTMLHelper::ReadMHTML() {
+  std::string buffer;
+  if (!base::ReadFileToString(mhtml_snapshot_path_, &buffer)) {
+    LOG(ERROR) << "Failed to read " << mhtml_snapshot_path_;
+    ReportFailure("Failed to read MHTML file");
+    return;
+  }
+  std::unique_ptr<std::string> buffer_ptr =
+      std::make_unique<std::string>(buffer);
+  ReportSuccess(std::move(buffer_ptr));
+}
+
+void DevToolsMHTMLHelper::ReportFailure(const std::string& message) {
+  if (!BrowserThread::CurrentlyOn(BrowserThread::UI)) {
+    base::PostTaskWithTraits(
+        FROM_HERE, {BrowserThread::UI},
+        base::BindOnce(&DevToolsMHTMLHelper::ReportFailure, this, message));
+    return;
+  }
+  if (message.empty())
+    callback_->sendFailure(Response::InternalError());
+  else
+    callback_->sendFailure(Response::Error(message));
+}
+
+void DevToolsMHTMLHelper::ReportSuccess(
+    std::unique_ptr<std::string> mhtml_snapshot) {
+  if (!BrowserThread::CurrentlyOn(BrowserThread::UI)) {
+    base::PostTaskWithTraits(FROM_HERE, {BrowserThread::UI},
+                             base::BindOnce(&DevToolsMHTMLHelper::ReportSuccess,
+                                            this, std::move(mhtml_snapshot)));
+    return;
+  }
+  callback_->sendSuccess(*mhtml_snapshot);
+}
+
+}  // namespace protocol
+}  // namespace content

content/browser/devtools/protocol/devtools_mhtml_helper.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CONTENT_BROWSER_DEVTOOLS_PROTOCOL_DEVTOOLS_MHTML_HELPER_H_
+#define CONTENT_BROWSER_DEVTOOLS_PROTOCOL_DEVTOOLS_MHTML_HELPER_H_
+
+#include "content/browser/devtools/protocol/page_handler.h"
+#include "storage/browser/blob/shareable_file_reference.h"
+
+namespace content {
+namespace protocol {
+
+class DevToolsMHTMLHelper
+    : public base::RefCountedThreadSafe<DevToolsMHTMLHelper> {
+ public:
+  static void Capture(
+      base::WeakPtr<PageHandler> page_handler,
+      std::unique_ptr<PageHandler::CaptureSnapshotCallback> callback);
+
+ private:
+  DevToolsMHTMLHelper(
+      base::WeakPtr<PageHandler> page_handler,
+      std::unique_ptr<PageHandler::CaptureSnapshotCallback> callback);
+  ~DevToolsMHTMLHelper();
+
+  void CreateTemporaryFile();
+  void TemporaryFileCreatedOnIO();
+  void TemporaryFileCreatedOnUI();
+  void MHTMLGeneratedOnUI(int64_t mhtml_file_size);
+  void ReadMHTML();
+  void ReportFailure(const std::string& message);
+  void ReportSuccess(std::unique_ptr<std::string> mhtml_snapshot);
+
+  base::WeakPtr<PageHandler> page_handler_;
+  std::unique_ptr<PageHandler::CaptureSnapshotCallback> callback_;
+  scoped_refptr<storage::ShareableFileReference> mhtml_file_;
+  base::FilePath mhtml_snapshot_path_;
+
+  friend class base::RefCountedThreadSafe<DevToolsMHTMLHelper>;
+};
+
+}  // namespace protocol
+}  // namespace content
+
+#endif  // CONTENT_BROWSER_DEVTOOLS_PROTOCOL_DEVTOOLS_MHTML_HELPER_H_

content/browser/devtools/protocol/page_handler.cc

@@ -25,6 +25,7 @@
 #include "content/browser/devtools/devtools_agent_host_impl.h"
 #include "content/browser/devtools/protocol/devtools_download_manager_delegate.h"
 #include "content/browser/devtools/protocol/devtools_download_manager_helper.h"
+#include "content/browser/devtools/protocol/devtools_mhtml_helper.h"
 #include "content/browser/devtools/protocol/emulation_handler.h"
 #include "content/browser/frame_host/navigation_request.h"
 #include "content/browser/frame_host/navigator.h"
@@ -66,6 +67,7 @@ namespace protocol {
 
 namespace {
 
+constexpr const char* kMhtml = "mhtml";
 constexpr const char* kPng = "png";
 constexpr const char* kJpeg = "jpeg";
 constexpr int kDefaultScreenshotQuality = 80;
@@ -589,6 +591,17 @@ Response PageHandler::NavigateToHistoryEntry(int entry_id) {
   return Response::InvalidParams("No entry with passed id");
 }
 
+void PageHandler::CaptureSnapshot(
+    Maybe<std::string> format,
+    std::unique_ptr<CaptureSnapshotCallback> callback) {
+  std::string snapshot_format = format.fromMaybe(kMhtml);
+  if (snapshot_format != kMhtml) {
+    callback->sendFailure(Response::Error("Unsupported snapshot format"));
+    return;
+  }
+  DevToolsMHTMLHelper::Capture(weak_factory_.GetWeakPtr(), std::move(callback));
+}
+
 void PageHandler::CaptureScreenshot(
     Maybe<std::string> format,
     Maybe<int> quality,

content/browser/devtools/protocol/page_handler.h

@@ -86,6 +86,7 @@ class PageHandler : public DevToolsDomainHandler,
                                  JavaScriptDialogCallback callback);
   void DidCloseJavaScriptDialog(bool success, const base::string16& user_input);
   void NavigationReset(NavigationRequest* navigation_request);
+  WebContentsImpl* GetWebContents();
 
   Response Enable() override;
   Response Disable() override;
@@ -114,6 +115,9 @@ class PageHandler : public DevToolsDomainHandler,
       Maybe<Page::Viewport> clip,
       Maybe<bool> from_surface,
       std::unique_ptr<CaptureScreenshotCallback> callback) override;
+  void CaptureSnapshot(
+      Maybe<std::string> format,
+      std::unique_ptr<CaptureSnapshotCallback> callback) override;
   void PrintToPDF(Maybe<bool> landscape,
                   Maybe<bool> display_header_footer,
                   Maybe<bool> print_background,
@@ -156,7 +160,6 @@ class PageHandler : public DevToolsDomainHandler,
  private:
   enum EncodingFormat { PNG, JPEG };
 
-  WebContentsImpl* GetWebContents();
   void NotifyScreencastVisibility(bool visible);
   void InnerSwapCompositorFrame();
   void OnFrameFromVideoConsumer(scoped_refptr<media::VideoFrame> frame);
@@ -219,6 +222,7 @@ class PageHandler : public DevToolsDomainHandler,
   scoped_refptr<DevToolsDownloadManagerDelegate> download_manager_delegate_;
   base::flat_map<base::UnguessableToken, std::unique_ptr<NavigateCallback>>
       navigate_callbacks_;
+
   base::WeakPtrFactory<PageHandler> weak_factory_;
 
   DISALLOW_COPY_AND_ASSIGN(PageHandler);

content/browser/devtools/protocol_config.json

@@ -49,9 +49,9 @@
                 "domain": "Page",
                 "include": ["enable", "disable", "reload", "navigate", "stopLoading", "getNavigationHistory", "navigateToHistoryEntry", "captureScreenshot",
                     "startScreencast", "stopScreencast", "screencastFrameAck", "handleJavaScriptDialog", "setColorPickerEnabled", "requestAppBanner",
-                    "printToPDF", "bringToFront", "setDownloadBehavior", "getAppManifest", "crash", "close", "setWebLifecycleState"],
+                    "printToPDF", "bringToFront", "setDownloadBehavior", "getAppManifest", "crash", "close", "setWebLifecycleState", "captureSnapshot"],
                 "include_events": ["colorPicked", "interstitialShown", "interstitialHidden", "javascriptDialogOpening", "javascriptDialogClosed", "screencastVisibilityChanged", "screencastFrame"],
-                "async": ["captureScreenshot", "printToPDF", "navigate", "getAppManifest", "reload"]
+                "async": ["captureScreenshot", "printToPDF", "navigate", "getAppManifest", "reload", "captureSnapshot"]
             },
             {
                 "domain": "Runtime",

third_party/WebKit/LayoutTests/inspector-protocol/page/capture-snapshot-expected.txt

+Tests capturing MHTML snapshots.
+
+Capturing without specified format:
+From: <Saved by Blink>
<Snapshot-Content-Location: >Subject: 
<Date: ><MIME-Version: >Content-Type: multipart/related;
	type="text/html";
<boundary="----MultipartBoundary-->

<------MultipartBoundary-->Content-Type: text/html
<Content-ID: ><Content-Transfer-Encoding: ><Content-Location: >
<html><head><meta http-equiv=3D"Content-Type" content=3D"text/html; charset=
=3Dwindows-1252"></head><body><div id=3D"x" class=3D"container">
      <p style=3D"color: red">Text</p>
     =20
    </div>
    </body></html>
<------MultipartBoundary-->
+
+Capturing with format: foo
+{
+    error : {
+        code : -32000
+        message : Unsupported snapshot format
+    }
+    id : <number>
+    sessionId : <string>
+}
+

third_party/WebKit/LayoutTests/inspector-protocol/page/capture-snapshot.js

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+(async function(testRunner) {
+  var {page, session, dp} = await testRunner.startHTML(
+    `
+    <div id="x" class="container">
+      <p style="color: red">Text</p>
+      <script>var foo;</script>
+    </div>
+    `, 'Tests capturing MHTML snapshots.');
+
+  await dp.Page.enable();
+
+  testRunner.log(`\nCapturing without specified format:`);
+  testRunner.log(formatResult(await dp.Page.captureSnapshot()));
+
+  testRunner.log(`\nCapturing with format: foo`);
+  testRunner.log(formatResult(await dp.Page.captureSnapshot({format: 'foo'})));
+  testRunner.completeTest();
+
+  function formatResult(result) {
+    const data = result.result ? result.result.data : null;
+    if (!data)
+      return result;
+
+    const ignoredPrefixes = [
+      'Snapshot-Content-Location: ',
+      'Subject: ',
+      'Date: ',
+      'MIME-Version: ',
+      'boundary="----MultipartBoundary--',
+      '------MultipartBoundary--',
+      'Content-ID: ',
+      'Content-Transfer-Encoding: ',
+      'Content-Location: ',
+    ];
+    let cleanData = '';
+    for (const line of data.split('\n')) {
+      let cleanLine = line;
+      for (const prefix of ignoredPrefixes) {
+        if (line.trim().startsWith(prefix)) {
+          cleanLine = `<${prefix}>`;
+          continue;
+        }
+      }
+      cleanData += cleanLine;
+    }
+    return cleanData;
+  }
+})

third_party/blink/renderer/core/inspector/browser_protocol.pdl

@@ -4928,6 +4928,17 @@ domain Page
       # Base64-encoded image data.
       binary data
 
+  # Returns a snapshot of the page as a string. For MHTML format, the serialization includes
+  # iframes, shadow DOM, external resources, and element-inline styles.
+  experimental command captureSnapshot
+    parameters
+      # Format (defaults to mhtml).
+      optional enum format
+        mhtml
+    returns
+      # Serialized page data.
+      string data
+
   # Clears the overriden device metrics.
   experimental deprecated command clearDeviceMetricsOverride
     # Use 'Emulation.clearDeviceMetricsOverride' instead

third_party/blink/renderer/core/inspector/inspector_protocol_config.json

@@ -74,7 +74,7 @@
             {
                 "domain": "Page",
                 "exclude": ["getNavigationHistory", "navigateToHistoryEntry", "captureScreenshot", "screencastFrameAck", "handleJavaScriptDialog", "setColorPickerEnabled",
-                            "getAppManifest", "requestAppBanner", "setControlNavigations", "processNavigation", "printToPDF", "bringToFront", "setDownloadBehavior", "navigate", "crash", "close", "setWebLifecycleState"],
+                            "getAppManifest", "requestAppBanner", "setControlNavigations", "processNavigation", "printToPDF", "bringToFront", "setDownloadBehavior", "navigate", "crash", "close", "setWebLifecycleState", "captureSnapshot"],
                 "async": ["getResourceContent", "searchInResource"],
                 "exclude_events": ["screencastFrame", "screencastVisibilityChanged", "colorPicked", "interstitialShown", "interstitialHidden", "javascriptDialogOpening", "javascriptDialogClosed", "navigationRequested"]
             },