[Offline pages] Add PageRenovator for script selection and injection.

Implements PageRenovator which is responsible for picking renovations,
injecting them into a page, and notifying the user of script
completion.

Design doc:
https://docs.google.com/document/d/1WuwRJaxBKAwVpeAHG0xhq5uRxMTB-j4LFleT4QEqANU

Bug: 736933
Change-Id: Icbd8b494d75298ed355f9666971ecf918b441d2e
Reviewed-on: https://chromium-review.googlesource.com/585907Reviewed-by: Paweł Hajdan Jr. <phajdan.jr@chromium.org>
Reviewed-by: Jochen Eisinger <jochen@chromium.org>
Reviewed-by: Yafei Duan <romax@chromium.org>
Reviewed-by: Justin DeWitt <dewittj@chromium.org>
Commit-Queue: Collin Baker <collinbaker@google.com>
Cr-Commit-Position: refs/heads/master@{#491175}

components/BUILD.gn

@@ -402,6 +402,7 @@ if (!is_ios) {
       "dom_distiller/content/browser/distillable_page_utils_browsertest.cc",
       "dom_distiller/content/browser/distiller_page_web_contents_browsertest.cc",
       "dom_distiller/content/browser/test/dom_distiller_js_browsertest.cc",
+      "offline_pages/content/renovations/test/page_renovator_browsertest.cc",
       "password_manager/content/renderer/credential_manager_client_browsertest.cc",
       "security_state/content/content_utils_browsertest.cc",
     ]
@@ -424,6 +425,8 @@ if (!is_ios) {
       "//components/dom_distiller/content/browser",
       "//components/dom_distiller/core",
       "//components/dom_distiller/core:test_support",
+      "//components/offline_pages/content/renovations",
+      "//components/offline_pages/core/renovations",
       "//components/password_manager/content/browser",
       "//components/password_manager/content/renderer",
       "//components/security_state/content",

components/offline_pages/content/renovations/BUILD.gn

+# Copyright 2017 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+if (is_android) {
+  import("//build/config/android/rules.gni")
+}
+
+source_set("renovations") {
+  sources = [
+    "render_frame_script_injector.cc",
+    "render_frame_script_injector.h",
+  ]
+
+  deps = [
+    "//components/offline_pages/core/renovations",
+    "//content/public/browser",
+  ]
+}

components/offline_pages/content/renovations/DEPS

+include_rules = [
+  "+content/public/browser",
+  "+content/public/common",
+]

components/offline_pages/content/renovations/render_frame_script_injector.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/offline_pages/content/renovations/render_frame_script_injector.h"
+
+#include <utility>
+
+#include "base/callback.h"
+#include "base/logging.h"
+#include "base/values.h"
+#include "content/public/browser/render_frame_host.h"
+#include "content/public/common/isolated_world_ids.h"
+
+namespace offline_pages {
+
+RenderFrameScriptInjector::RenderFrameScriptInjector(
+    content::RenderFrameHost* render_frame_host,
+    int isolated_world_id)
+    : render_frame_host_(render_frame_host),
+      isolated_world_id_(isolated_world_id) {
+  DCHECK(render_frame_host_);
+  DCHECK(isolated_world_id_ > content::ISOLATED_WORLD_ID_GLOBAL &&
+         isolated_world_id_ < content::ISOLATED_WORLD_ID_MAX);
+}
+
+void RenderFrameScriptInjector::Inject(base::string16 script,
+                                       ResultCallback callback) {
+  // Must create proxy callback since ExecuteJavaScriptInIsolatedWorld
+  // takes a |const base::Value*| argument instead of a |const
+  // base::Value&|.
+  base::RepeatingCallback<void(const base::Value*)> proxy_callback =
+      base::BindRepeating(
+          [](ResultCallback user_callback, const base::Value* result) {
+            base::Value new_result = result ? *result : base::Value();
+            if (user_callback)
+              user_callback.Run(new_result);
+          },
+          callback);
+
+  // |render_frame_host_| should still be alive if the
+  // caller is using this class correctly.
+  DCHECK(render_frame_host_);
+  render_frame_host_->ExecuteJavaScriptInIsolatedWorld(
+      script, std::move(proxy_callback), isolated_world_id_);
+}
+
+}  // namespace offline_pages

components/offline_pages/content/renovations/render_frame_script_injector.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_OFFLINE_PAGES_CONTENT_RENOVATIONS_RENDER_FRAME_SCRIPT_INJECTOR_H_
+#define COMPONENTS_OFFLINE_PAGES_CONTENT_RENOVATIONS_RENDER_FRAME_SCRIPT_INJECTOR_H_
+
+#include "components/offline_pages/core/renovations/script_injector.h"
+
+namespace content {
+class RenderFrameHost;
+}
+
+namespace offline_pages {
+
+// ScriptInjector for running scripts in a RenderFrame within a given isolated
+// world.
+class RenderFrameScriptInjector : public ScriptInjector {
+ public:
+  ~RenderFrameScriptInjector() override = default;
+
+  // The |render_frame_host| is expected to outlive this
+  // RenderFrameScriptInjector instance.
+  RenderFrameScriptInjector(content::RenderFrameHost* render_frame_host,
+                            int isolated_world_id);
+
+  // ScriptInjector implementation.
+  void Inject(base::string16 script, ResultCallback callback) override;
+
+ private:
+  content::RenderFrameHost* render_frame_host_;
+  int isolated_world_id_;
+};
+
+}  // namespace offline_pages
+
+#endif  // COMPONENTS_OFFLINE_PAGES_CONTENT_RENOVATIONS_RENDER_FRAME_SCRIPT_INJECTOR_H_

components/offline_pages/content/renovations/test/DEPS

+include_rules = [
+  "+content/public/test",
+  "+content/shell/browser",
+  "+net/test/embedded_test_server",
+]

components/offline_pages/content/renovations/test/page_renovator_browsertest.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/offline_pages/core/renovations/page_renovator.h"
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "base/files/file_path.h"
+#include "base/run_loop.h"
+#include "base/strings/utf_string_conversions.h"
+#include "base/values.h"
+#include "components/offline_pages/content/renovations/render_frame_script_injector.h"
+#include "components/offline_pages/core/renovations/page_renovation_loader.h"
+#include "content/public/browser/browser_thread.h"
+#include "content/public/browser/web_contents.h"
+#include "content/public/common/isolated_world_ids.h"
+#include "content/public/test/content_browser_test.h"
+#include "content/public/test/content_browser_test_utils.h"
+#include "content/public/test/test_utils.h"
+#include "content/shell/browser/shell.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "url/gurl.h"
+
+namespace offline_pages {
+
+namespace {
+
+const char kDocRoot[] = "components/test/data/offline_pages";
+const char kTestPageURL[] = "/renovator_test_page.html";
+const char kTestRenovationScript[] =
+    R"*(function foo() {
+      var node = document.getElementById('foo');
+      node.innerHTML = 'Good';
+    }
+    function bar() {
+      var node = document.getElementById('bar');
+      node.parentNode.removeChild(node);
+    }
+    function always() {
+      var node = document.getElementById('always');
+      node.parentNode.removeChild(node);
+    }
+    var map_renovations = {'foo':foo, 'bar':bar, 'always':always};
+    function run_renovations(idlist) {
+      for (var id of idlist) {
+        map_renovations[id]();
+      }
+    })*";
+
+// Scripts to check whether each renovation ran in the test page.
+const char kCheckFooScript[] =
+    "document.getElementById('foo').innerHTML == 'Good'";
+const char kCheckBarScript[] = "document.getElementById('bar') == null";
+const char kCheckAlwaysScript[] = "document.getElementById('always') == null";
+
+// Renovation that should only run on pages from URL foo.bar
+class FooPageRenovation : public PageRenovation {
+ public:
+  bool ShouldRun(const GURL& url) const override {
+    return url.host() == "foo.bar";
+  }
+  std::string GetID() const override { return "foo"; }
+};
+
+// Renovation that should only run on pages from URL bar.qux
+class BarPageRenovation : public PageRenovation {
+ public:
+  bool ShouldRun(const GURL& url) const override {
+    return url.host() == "bar.qux";
+  }
+  std::string GetID() const override { return "bar"; }
+};
+
+// Renovation that should run on every page.
+class AlwaysRenovation : public PageRenovation {
+ public:
+  bool ShouldRun(const GURL& url) const override { return true; }
+  std::string GetID() const override { return "always"; }
+};
+
+}  // namespace
+
+class PageRenovatorBrowserTest : public content::ContentBrowserTest {
+ public:
+  void SetUpOnMainThread() override;
+
+  void QuitRunLoop();
+
+ protected:
+  net::EmbeddedTestServer test_server_;
+  content::RenderFrameHost* render_frame_;
+  std::unique_ptr<PageRenovationLoader> page_renovation_loader_;
+  std::unique_ptr<PageRenovator> page_renovator_;
+
+  std::unique_ptr<base::RunLoop> run_loop_;
+};
+
+void PageRenovatorBrowserTest::SetUpOnMainThread() {
+  // Serve our test HTML.
+  test_server_.ServeFilesFromSourceDirectory(kDocRoot);
+  ASSERT_TRUE(test_server_.Start());
+
+  // Navigate to test page.
+  GURL url = test_server_.GetURL(kTestPageURL);
+  content::NavigateToURL(shell(), url);
+  render_frame_ = shell()->web_contents()->GetMainFrame();
+
+  // Initialize renovations.
+  std::vector<std::unique_ptr<PageRenovation>> renovations;
+  renovations.push_back(base::MakeUnique<FooPageRenovation>());
+  renovations.push_back(base::MakeUnique<BarPageRenovation>());
+  renovations.push_back(base::MakeUnique<AlwaysRenovation>());
+
+  page_renovation_loader_.reset(new PageRenovationLoader);
+  page_renovation_loader_->SetSourceForTest(
+      base::ASCIIToUTF16(kTestRenovationScript));
+  page_renovation_loader_->SetRenovationsForTest(std::move(renovations));
+
+  auto script_injector = base::MakeUnique<RenderFrameScriptInjector>(
+      render_frame_, content::ISOLATED_WORLD_ID_CONTENT_END);
+  GURL fake_url("http://foo.bar/");
+  page_renovator_.reset(new PageRenovator(
+      page_renovation_loader_.get(), std::move(script_injector), fake_url));
+
+  run_loop_.reset(new base::RunLoop);
+}
+
+void PageRenovatorBrowserTest::QuitRunLoop() {
+  base::Closure quit_task =
+      content::GetDeferredQuitTaskForRunLoop(run_loop_.get());
+  content::BrowserThread::PostTask(content::BrowserThread::UI, FROM_HERE,
+                                   quit_task);
+}
+
+IN_PROC_BROWSER_TEST_F(PageRenovatorBrowserTest, CorrectRenovationsRun) {
+  // This should run FooPageRenovation and AlwaysRenovation, but not
+  // BarPageRenovation.
+  page_renovator_->RunRenovations(base::Bind(
+      &PageRenovatorBrowserTest::QuitRunLoop, base::Unretained(this)));
+  content::RunThisRunLoop(run_loop_.get());
+
+  // Check that correct modifications were made to the page.
+  std::unique_ptr<base::Value> fooResult =
+      content::ExecuteScriptAndGetValue(render_frame_, kCheckFooScript);
+  std::unique_ptr<base::Value> barResult =
+      content::ExecuteScriptAndGetValue(render_frame_, kCheckBarScript);
+  std::unique_ptr<base::Value> alwaysResult =
+      content::ExecuteScriptAndGetValue(render_frame_, kCheckAlwaysScript);
+
+  ASSERT_TRUE(fooResult.get() != nullptr);
+  ASSERT_TRUE(barResult.get() != nullptr);
+  ASSERT_TRUE(alwaysResult.get() != nullptr);
+  EXPECT_TRUE(fooResult->GetBool());
+  EXPECT_FALSE(barResult->GetBool());
+  EXPECT_TRUE(alwaysResult->GetBool());
+}
+
+// TODO(collinbaker): add test for Wikipedia renovation here.
+
+}  // namespace offline_pages

components/offline_pages/core/renovations/BUILD.gn

@@ -11,10 +11,14 @@ static_library("renovations") {
     "page_renovation.h",
     "page_renovation_loader.cc",
     "page_renovation_loader.h",
+    "page_renovator.cc",
+    "page_renovator.h",
+    "script_injector.h",
   ]
 
   deps = [
     "//base",
+    "//components/offline_pages/core",
     "//url",
   ]
 }
@@ -23,11 +27,15 @@ source_set("unit_tests") {
   testonly = true
   sources = [
     "page_renovation_loader_unittest.cc",
+    "page_renovator_unittest.cc",
   ]
 
   deps = [
     ":renovations",
     "//base",
+    "//base/test:test_support",
+    "//testing/gmock",
     "//testing/gtest",
+    "//url",
   ]
 }

components/offline_pages/core/renovations/page_renovation_loader.h

@@ -34,17 +34,15 @@ class PageRenovationLoader {
     return renovations_;
   }
 
- private:
-  // Called to load JavaScript source from storage.
-  bool LoadSource();
-
-  // Set combined_source_ for unit tests.
+  // Methods for testing.
   void SetSourceForTest(base::string16 combined_source);
-
-  // Set renovations_ for unit tests.
   void SetRenovationsForTest(
       std::vector<std::unique_ptr<PageRenovation>> renovations);
 
+ private:
+  // Called to load JavaScript source from storage.
+  bool LoadSource();
+
   // List of registered page renovations
   std::vector<std::unique_ptr<PageRenovation>> renovations_;
 
@@ -53,9 +51,6 @@ class PageRenovationLoader {
   // Contains JavaScript source.
   base::string16 combined_source_;
 
-  // Friend class for unit tests.
-  friend class PageRenovationLoaderTest;
-
   DISALLOW_COPY_AND_ASSIGN(PageRenovationLoader);
 };
 

components/offline_pages/core/renovations/page_renovator.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/offline_pages/core/renovations/page_renovator.h"
+
+#include <utility>
+
+#include "base/bind.h"
+#include "base/callback.h"
+#include "base/logging.h"
+
+namespace offline_pages {
+
+PageRenovator::PageRenovator(PageRenovationLoader* renovation_loader,
+                             std::unique_ptr<ScriptInjector> script_injector,
+                             const GURL& request_url)
+    : renovation_loader_(renovation_loader),
+      script_injector_(std::move(script_injector)) {
+  DCHECK(renovation_loader);
+
+  PrepareScript(request_url);
+}
+
+PageRenovator::~PageRenovator() {}
+
+void PageRenovator::RunRenovations(base::Closure callback) {
+  // Prepare callback and inject combined script.
+  base::RepeatingCallback<void(const base::Value&)> cb = base::Bind(
+      [](base::Closure callback, const base::Value&) {
+        if (callback)
+          callback.Run();
+      },
+      std::move(callback));
+
+  script_injector_->Inject(script_, cb);
+}
+
+void PageRenovator::PrepareScript(const GURL& url) {
+  std::vector<std::string> renovation_keys;
+
+  // Pick which renovations to run.
+  for (const std::unique_ptr<PageRenovation>& renovation :
+       renovation_loader_->renovations()) {
+    if (renovation->ShouldRun(url)) {
+      renovation_keys.push_back(renovation->GetID());
+    }
+  }
+
+  // Store combined renovation script. TODO(crbug.com/736933): handle
+  // failed GetRenovationScript call.
+  renovation_loader_->GetRenovationScript(renovation_keys, &script_);
+}
+
+}  // namespace offline_pages

components/offline_pages/core/renovations/page_renovator.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_OFFLINE_PAGES_CORE_RENOVATIONS_PAGE_RENOVATOR_H_
+#define COMPONENTS_OFFLINE_PAGES_CORE_RENOVATIONS_PAGE_RENOVATOR_H_
+
+#include <memory>
+
+#include "base/callback.h"
+#include "base/memory/weak_ptr.h"
+#include "base/strings/string16.h"
+#include "base/time/time.h"
+#include "components/offline_pages/core/renovations/page_renovation_loader.h"
+#include "components/offline_pages/core/renovations/script_injector.h"
+#include "components/offline_pages/core/snapshot_controller.h"
+
+namespace offline_pages {
+
+// This class runs renovations in a page being offlined. Upon
+// construction, it determines which renovations to run. The user
+// should then call RunRenovations when the page is sufficiently
+// loaded. When complete (if ever) the passed CompletionCallback will
+// be called.
+class PageRenovator {
+ public:
+  using CompletionCallback = base::RepeatingClosure;
+
+  // |renovation_loader| should be owned by the user and is expected to
+  // outlive this PageRenovator instance.
+  PageRenovator(PageRenovationLoader* renovation_loader,
+                std::unique_ptr<ScriptInjector> script_injector,
+                const GURL& request_url);
+  ~PageRenovator();
+
+  // Run renovation scripts and call callback when they complete.
+  void RunRenovations(CompletionCallback callback);
+
+ private:
+  // This method is used in the constructor to determine which
+  // renovations to run and populate |script_| with the renovations.
+  void PrepareScript(const GURL& url);
+
+  PageRenovationLoader* renovation_loader_;
+  std::unique_ptr<ScriptInjector> script_injector_;
+  base::string16 script_;
+};
+
+}  // namespace offline_pages
+
+#endif  // COMPONENTS_OFFLINE_PAGES_CORE_RENOVATIONS_PAGE_RENOVATOR_H_

components/offline_pages/core/renovations/page_renovator_unittest.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/offline_pages/core/renovations/page_renovator.h"
+
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "base/bind.h"
+#include "base/memory/ptr_util.h"
+#include "base/test/mock_callback.h"
+#include "base/values.h"
+#include "components/offline_pages/core/renovations/script_injector.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include "url/gurl.h"
+
+namespace offline_pages {
+
+class PageRenovatorTest : public testing::Test {
+ public:
+  PageRenovatorTest();
+  ~PageRenovatorTest() override;
+
+ protected:
+  // ScriptInjector for testing PageRenovator. When Inject is called,
+  // it sets |script_injector_inject_called_| in the fixture to true,
+  // then calls the callback.
+  class FakeScriptInjector : public ScriptInjector {
+   public:
+    FakeScriptInjector(PageRenovatorTest* fixture);
+    ~FakeScriptInjector() override = default;
+
+    void Inject(base::string16 script, ResultCallback callback) override;
+
+   private:
+    PageRenovatorTest* fixture_;
+  };
+
+  // Creates a PageRenovator with simple defaults for testing.
+  void CreatePageRenovator(const GURL& url);
+
+  PageRenovationLoader page_renovation_loader_;
+  bool script_injector_inject_called_ = false;
+
+  // PageRenovator under test.
+  std::unique_ptr<PageRenovator> page_renovator_;
+};
+
+PageRenovatorTest::FakeScriptInjector::FakeScriptInjector(
+    PageRenovatorTest* fixture)
+    : fixture_(fixture) {}
+
+void PageRenovatorTest::FakeScriptInjector::Inject(base::string16 script,
+                                                   ResultCallback callback) {
+  if (callback)
+    callback.Run(base::Value());
+  fixture_->script_injector_inject_called_ = true;
+}
+
+PageRenovatorTest::PageRenovatorTest() {
+  // Set PageRenovationLoader to have empty script and Renovation list.
+  page_renovation_loader_.SetSourceForTest(base::string16());
+  page_renovation_loader_.SetRenovationsForTest(
+      std::vector<std::unique_ptr<PageRenovation>>());
+
+  page_renovator_ = base::MakeUnique<PageRenovator>(
+      &page_renovation_loader_, base::MakeUnique<FakeScriptInjector>(this),
+      GURL("example.com"));
+}
+
+PageRenovatorTest::~PageRenovatorTest() {}
+
+TEST_F(PageRenovatorTest, InjectsScript) {
+  EXPECT_FALSE(script_injector_inject_called_);
+  page_renovator_->RunRenovations(base::Closure());
+  EXPECT_TRUE(script_injector_inject_called_);
+}
+
+TEST_F(PageRenovatorTest, CallsCallback) {
+  base::MockCallback<base::Closure> mock_callback;
+  EXPECT_CALL(mock_callback, Run()).Times(1);
+
+  page_renovator_->RunRenovations(mock_callback.Get());
+}
+
+}  // namespace offline_pages

components/offline_pages/core/renovations/script_injector.h

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_OFFLINE_PAGES_CORE_RENOVATIONS_SCRIPT_INJECTOR_H_
+#define COMPONENTS_OFFLINE_PAGES_CORE_RENOVATIONS_SCRIPT_INJECTOR_H_
+
+#include "base/callback_forward.h"
+#include "base/strings/string16.h"
+
+namespace base {
+class Value;
+}  // namespace base
+
+namespace offline_pages {
+
+// Interface for injecting and running scripts in some
+// context. Inject() takes a string of JavaScript, injects it into the
+// context, then calls the ResultCallback with the expression value.
+class ScriptInjector {
+ public:
+  using ResultCallback = base::Callback<void(const base::Value&)>;
+
+  virtual ~ScriptInjector() = default;
+
+  // Takes a string of JavaScript, injects it into the context, then
+  // calls the ResultCallback with the expression value.
+  virtual void Inject(base::string16 script, ResultCallback callback) = 0;
+};
+
+}  // namespace offline_pages
+
+#endif  // COMPONENTS_OFFLINE_PAGES_CORE_RENOVATIONS_SCRIPT_INJECTOR_H_

components/test/data/offline_pages/renovator_test_page.html

+<html>
+  <body>
+    <div id="foo">Bad</div>
+    <div id="bar"></div>
+    <div id="always"></div>
+  </body>
+</html>