Renderer-side changes for NoState Prefetch

NoState Prefetch design doc:
https://docs.google.com/document/d/16VCYGGWau483IMSxODpg5faZny1FJ6vNK2v-BuM5EhU

This CL is the first step for NoState Prefetch implementation
in the renderer.

The code added in this CL is not called yet (it will be called
only when the PREFETCH_ONLY enum is sent through IPC).

The prefetching is implemented in the HTMLDocumentParser.
When prefetching, the parser runs in single-thread mode.
When feeded with HTML input, it runs the preload scanner on
it, and stops without actually building or executing the page.

BUG=632361

Review-Url: https://codereview.chromium.org/2172613002
Cr-Commit-Position: refs/heads/master@{#414406}

chrome/renderer/chrome_render_frame_observer.cc

@@ -170,7 +170,7 @@ void ChromeRenderFrameObserver::OnSetIsPrerendering(
 
     // The PrerenderHelper will destroy itself either after recording histograms
     // or on destruction of the RenderView.
-    new prerender::PrerenderHelper(render_frame());
+    new prerender::PrerenderHelper(render_frame(), mode);
   }
 }
 

chrome/renderer/prerender/prerender_helper.cc

@@ -34,18 +34,39 @@ void UpdateVisibilityState(content::RenderFrame* render_frame) {
 
 namespace prerender {
 
-PrerenderHelper::PrerenderHelper(content::RenderFrame* render_frame)
+PrerenderHelper::PrerenderHelper(content::RenderFrame* render_frame,
+                                 PrerenderMode prerender_mode)
     : content::RenderFrameObserver(render_frame),
-      content::RenderFrameObserverTracker<PrerenderHelper>(render_frame) {
+      content::RenderFrameObserverTracker<PrerenderHelper>(render_frame),
+      prerender_mode_(prerender_mode) {
+  DCHECK_NE(prerender_mode_, NO_PRERENDER);
   UpdateVisibilityState(render_frame);
 }
 
+PrerenderHelper::PrerenderHelper(content::RenderFrame* sub_frame)
+    : PrerenderHelper(sub_frame,
+                      GetPrerenderMode(
+                          sub_frame->GetRenderView()->GetMainRenderFrame())) {
+  DCHECK(!render_frame()->IsMainFrame());
+}
+
 PrerenderHelper::~PrerenderHelper() {
 }
 
 // static.
 bool PrerenderHelper::IsPrerendering(const content::RenderFrame* render_frame) {
-  return PrerenderHelper::Get(render_frame) != NULL;
+  return PrerenderHelper::GetPrerenderMode(render_frame) != NO_PRERENDER;
+}
+
+// static.
+PrerenderMode PrerenderHelper::GetPrerenderMode(
+    const content::RenderFrame* render_frame) {
+  PrerenderHelper* helper = PrerenderHelper::Get(render_frame);
+  if (!helper)
+    return NO_PRERENDER;
+
+  DCHECK_NE(helper->prerender_mode_, NO_PRERENDER);
+  return helper->prerender_mode_;
 }
 
 bool PrerenderHelper::OnMessageReceived(

chrome/renderer/prerender/prerender_helper.h

@@ -20,12 +20,21 @@ class PrerenderHelper
     : public content::RenderFrameObserver,
       public content::RenderFrameObserverTracker<PrerenderHelper> {
  public:
-  explicit PrerenderHelper(content::RenderFrame* render_frame);
+  PrerenderHelper(content::RenderFrame* render_frame,
+                  PrerenderMode prerender_mode);
+
+  // Must be called on a sub-frame. Inherits the prerender mode from the main
+  // frame.
+  explicit PrerenderHelper(content::RenderFrame* sub_frame);
+
   ~PrerenderHelper() override;
 
   // Returns true if |render_view| is currently prerendering.
   static bool IsPrerendering(const content::RenderFrame* render_frame);
 
+  static PrerenderMode GetPrerenderMode(
+      const content::RenderFrame* render_frame);
+
  private:
   // RenderFrameObserver implementation.
   bool OnMessageReceived(const IPC::Message& message) override;
@@ -33,6 +42,8 @@ class PrerenderHelper
 
   void OnSetIsPrerendering(PrerenderMode mode);
 
+  PrerenderMode prerender_mode_;
+
   DISALLOW_COPY_AND_ASSIGN(PrerenderHelper);
 };
 

chrome/renderer/prerender/prerenderer_client.cc

@@ -6,6 +6,7 @@
 
 #include "base/logging.h"
 #include "chrome/renderer/prerender/prerender_extra_data.h"
+#include "chrome/renderer/prerender/prerender_helper.h"
 #include "content/public/renderer/render_view.h"
 #include "third_party/WebKit/public/web/WebView.h"
 
@@ -34,6 +35,11 @@ void PrerendererClient::willAddPrerender(
                                                  render_view()->GetSize()));
 }
 
+bool PrerendererClient::isPrefetchOnly() {
+  return PrerenderHelper::GetPrerenderMode(
+             render_view()->GetMainRenderFrame()) == PREFETCH_ONLY;
+}
+
 void PrerendererClient::OnDestruct() {
   delete this;
 }

chrome/renderer/prerender/prerenderer_client.h

@@ -24,6 +24,7 @@ class PrerendererClient : public content::RenderViewObserver,
 
   // Implements blink::WebPrerendererClient
   void willAddPrerender(blink::WebPrerender* prerender) override;
+  bool isPrefetchOnly() override;
 };
 
 }  // namespace prerender

third_party/WebKit/Source/core/core.gypi

@@ -4211,6 +4211,7 @@
             'html/forms/StepRangeTest.cpp',
             'html/parser/AtomicHTMLTokenTest.cpp',
             'html/parser/CompactHTMLTokenTest.cpp',
+            'html/parser/HTMLDocumentParserTest.cpp',
             'html/parser/HTMLEntityParserTest.cpp',
             'html/parser/HTMLParserIdiomsTest.cpp',
             'html/parser/HTMLParserThreadTest.cpp',

third_party/WebKit/Source/core/dom/DecodedDataDocumentParser.h

@@ -26,13 +26,14 @@
 #ifndef DecodedDataDocumentParser_h
 #define DecodedDataDocumentParser_h
 
+#include "core/CoreExport.h"
 #include "core/dom/DocumentParser.h"
 #include <memory>
 
 namespace blink {
 class TextResourceDecoder;
 
-class DecodedDataDocumentParser : public DocumentParser {
+class CORE_EXPORT DecodedDataDocumentParser : public DocumentParser {
 public:
     // Only used by the XMLDocumentParser to communicate back to
     // XMLHttpRequest if the responseXML was well formed.

third_party/WebKit/Source/core/dom/Document.cpp

@@ -194,6 +194,7 @@
 #include "core/loader/FrameLoaderClient.h"
 #include "core/loader/ImageLoader.h"
 #include "core/loader/NavigationScheduler.h"
+#include "core/loader/PrerendererClient.h"
 #include "core/loader/appcache/ApplicationCacheHost.h"
 #include "core/observer/ResizeObserverController.h"
 #include "core/page/ChromeClient.h"
@@ -1326,6 +1327,15 @@ PageVisibilityState Document::pageVisibilityState() const
     return m_frame->page()->visibilityState();
 }
 
+bool Document::isPrefetchOnly() const
+{
+    if (!m_frame || !m_frame->page())
+        return false;
+
+    PrerendererClient* prerendererClient = PrerendererClient::from(m_frame->page());
+    return prerendererClient && prerendererClient->isPrefetchOnly();
+}
+
 String Document::visibilityState() const
 {
     return pageVisibilityStateString(pageVisibilityState());
@@ -2436,8 +2446,12 @@ DocumentParser* Document::implicitOpen(ParserSynchronizationPolicy parserSyncPol
 
     setCompatibilityMode(NoQuirksMode);
 
-    if (!threadedParsingEnabledForTesting())
+    if (!threadedParsingEnabledForTesting()) {
         parserSyncPolicy = ForceSynchronousParsing;
+    } else if (parserSyncPolicy == AllowAsynchronousParsing && isPrefetchOnly()) {
+        // Prefetch must be synchronous.
+        parserSyncPolicy = ForceSynchronousParsing;
+    }
 
     m_parserSyncPolicy = parserSyncPolicy;
     m_parser = createParser();

third_party/WebKit/Source/core/dom/Document.h

@@ -355,6 +355,11 @@ public:
     bool hidden() const;
     void didChangeVisibilityState();
 
+    // If the document is "prefetch only", it will not be fully contstructed,
+    // and should never be displayed. Only a few resources will be loaded and
+    // scanned, in order to warm up caches.
+    bool isPrefetchOnly() const;
+
     Node* adoptNode(Node* source, ExceptionState&);
 
     HTMLCollection* images();

third_party/WebKit/Source/core/dom/DocumentParser.h

@@ -24,6 +24,7 @@
 #ifndef DocumentParser_h
 #define DocumentParser_h
 
+#include "core/CoreExport.h"
 #include "platform/heap/Handle.h"
 #include "wtf/Forward.h"
 #include <memory>
@@ -36,7 +37,7 @@ class SegmentedString;
 class ScriptableDocumentParser;
 class TextResourceDecoder;
 
-class DocumentParser : public GarbageCollectedFinalized<DocumentParser> {
+class CORE_EXPORT DocumentParser : public GarbageCollectedFinalized<DocumentParser> {
 public:
     virtual ~DocumentParser();
     DECLARE_VIRTUAL_TRACE();

third_party/WebKit/Source/core/dom/ScriptableDocumentParser.h

@@ -26,13 +26,14 @@
 #ifndef ScriptableDocumentParser_h
 #define ScriptableDocumentParser_h
 
+#include "core/CoreExport.h"
 #include "core/dom/DecodedDataDocumentParser.h"
 #include "core/dom/ParserContentPolicy.h"
 #include "wtf/text/TextPosition.h"
 
 namespace blink {
 
-class ScriptableDocumentParser : public DecodedDataDocumentParser {
+class CORE_EXPORT ScriptableDocumentParser : public DecodedDataDocumentParser {
 public:
     // Only used by Document::open for deciding if its safe to act on a
     // JavaScript document.open() call right now, or it should be ignored.

third_party/WebKit/Source/core/html/parser/HTMLDocumentParser.cpp

@@ -131,6 +131,9 @@ HTMLDocumentParser::HTMLDocumentParser(Document& document, ParserContentPolicy c
     , m_triedLoadingLinkHeaders(false)
 {
     ASSERT(shouldUseThreading() || (m_token && m_tokenizer));
+    // Threading is not allowed in prefetch mode.
+    DCHECK(!document.isPrefetchOnly() || !shouldUseThreading());
+
     ThreadState::current()->registerPreFinalizer(this);
 }
 
@@ -816,6 +819,18 @@ void HTMLDocumentParser::append(const String& inputSource)
     TRACE_EVENT1(TRACE_DISABLED_BY_DEFAULT("blink.debug"), "HTMLDocumentParser::append", "size", inputSource.length());
     const SegmentedString source(inputSource);
 
+    if (document()->isPrefetchOnly()) {
+        if (!m_preloadScanner)
+            m_preloadScanner = createPreloadScanner();
+
+        m_preloadScanner->appendToEnd(source);
+        // TODO(droger): Set the LOAD_PREFETCH flags on the requests.
+        m_preloadScanner->scanAndPreload(m_preloader.get(), document()->validBaseElementURL(), nullptr);
+
+        // Return after the preload scanner, do not actually parse the document.
+        return;
+    }
+
     if (m_preloadScanner) {
         if (m_input.current().isEmpty() && !isWaitingForScripts()) {
             // We have parsed until the end of the current input and so are now moving ahead of the preload scanner.

third_party/WebKit/Source/core/html/parser/HTMLDocumentParser.h

@@ -27,6 +27,7 @@
 #define HTMLDocumentParser_h
 
 #include "bindings/core/v8/DocumentWriteEvaluator.h"
+#include "core/CoreExport.h"
 #include "core/dom/ParserContentPolicy.h"
 #include "core/dom/ScriptableDocumentParser.h"
 #include "core/fetch/ResourceClient.h"
@@ -69,7 +70,7 @@ class HTMLTreeBuilder;
 class PumpSession;
 class TokenizedChunkQueue;
 
-class HTMLDocumentParser :  public ScriptableDocumentParser, private HTMLScriptRunnerHost {
+class CORE_EXPORT HTMLDocumentParser :  public ScriptableDocumentParser, private HTMLScriptRunnerHost {
     USING_GARBAGE_COLLECTED_MIXIN(HTMLDocumentParser);
     USING_PRE_FINALIZER(HTMLDocumentParser, dispose);
 public:
@@ -88,6 +89,9 @@ public:
 
     static void parseDocumentFragment(const String&, DocumentFragment*, Element* contextElement, ParserContentPolicy = AllowScriptingContent);
 
+    // Exposed for testing.
+    HTMLScriptRunnerHost* asHTMLScriptRunnerHostForTesting() { return this; }
+
     HTMLTokenizer* tokenizer() const { return m_tokenizer.get(); }
 
     TextPosition textPosition() const final;

third_party/WebKit/Source/core/html/parser/HTMLDocumentParserTest.cpp

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "core/html/parser/HTMLDocumentParser.h"
+
+#include "core/html/HTMLDocument.h"
+#include "core/html/parser/TextResourceDecoder.h"
+#include "core/loader/PrerendererClient.h"
+#include "core/loader/TextResourceDecoderBuilder.h"
+#include "core/testing/DummyPageHolder.h"
+#include "testing/gtest/include/gtest/gtest.h"
+#include <memory>
+
+namespace blink {
+
+namespace {
+
+class TestPrerendererClient :
+    public GarbageCollected<TestPrerendererClient>, public PrerendererClient {
+USING_GARBAGE_COLLECTED_MIXIN(TestPrerendererClient);
+
+public:
+    TestPrerendererClient(bool isPrefetchOnly)
+        : m_isPrefetchOnly(isPrefetchOnly)
+    {
+    }
+
+private:
+    void willAddPrerender(Prerender*) override {};
+    bool isPrefetchOnly() override { return m_isPrefetchOnly; }
+
+    bool m_isPrefetchOnly;
+};
+
+class HTMLDocumentParserTest : public testing::Test {
+protected:
+    HTMLDocumentParserTest()
+        : m_dummyPageHolder(DummyPageHolder::create())
+    {
+    }
+
+    HTMLDocumentParser* createParser(HTMLDocument& document)
+    {
+        HTMLDocumentParser* parser =
+            HTMLDocumentParser::create(document, ForceSynchronousParsing);
+        TextResourceDecoderBuilder decoderBuilder("text/html", nullAtom);
+        std::unique_ptr<TextResourceDecoder> decoder(
+            decoderBuilder.buildFor(&document));
+        parser->setDecoder(std::move(decoder));
+        return parser;
+    }
+
+    std::unique_ptr<DummyPageHolder> m_dummyPageHolder;
+};
+
+} // namespace
+
+TEST_F(HTMLDocumentParserTest, AppendPrefetch)
+{
+    HTMLDocument& document = toHTMLDocument(m_dummyPageHolder->document());
+    providePrerendererClientTo(*document.page(),
+        new TestPrerendererClient(true));
+    EXPECT_TRUE(document.isPrefetchOnly());
+    HTMLDocumentParser* parser = createParser(document);
+
+    const char bytes[] = "<ht";
+    parser->appendBytes(bytes, sizeof(bytes));
+    // The bytes are forwarded to the preload scanner, not to the tokenizer.
+    HTMLScriptRunnerHost* scriptRunnerHost =
+        parser->asHTMLScriptRunnerHostForTesting();
+    EXPECT_TRUE(scriptRunnerHost->hasPreloadScanner());
+    EXPECT_EQ(HTMLTokenizer::DataState, parser->tokenizer()->getState());
+}
+
+TEST_F(HTMLDocumentParserTest, AppendNoPrefetch)
+{
+    HTMLDocument& document = toHTMLDocument(m_dummyPageHolder->document());
+    EXPECT_FALSE(document.isPrefetchOnly());
+    // Use ForceSynchronousParsing to allow calling append().
+    HTMLDocumentParser* parser = createParser(document);
+
+    const char bytes[] = "<ht";
+    parser->appendBytes(bytes, sizeof(bytes));
+    // The bytes are forwarded to the tokenizer.
+    HTMLScriptRunnerHost* scriptRunnerHost =
+        parser->asHTMLScriptRunnerHostForTesting();
+    EXPECT_FALSE(scriptRunnerHost->hasPreloadScanner());
+    EXPECT_EQ(HTMLTokenizer::TagNameState, parser->tokenizer()->getState());
+}
+
+} // namespace blink

third_party/WebKit/Source/core/html/parser/HTMLScriptRunnerHost.h

@@ -26,6 +26,7 @@
 #ifndef HTMLScriptRunnerHost_h
 #define HTMLScriptRunnerHost_h
 
+#include "core/CoreExport.h"
 #include "wtf/Forward.h"
 
 namespace blink {
@@ -34,7 +35,7 @@ class HTMLInputStream;
 class Resource;
 class Visitor;
 
-class HTMLScriptRunnerHost : public GarbageCollectedMixin {
+class CORE_EXPORT HTMLScriptRunnerHost : public GarbageCollectedMixin {
 public:
     virtual ~HTMLScriptRunnerHost() { }
     DEFINE_INLINE_VIRTUAL_TRACE() { }

third_party/WebKit/Source/core/loader/PrerendererClient.h

@@ -44,6 +44,7 @@ class Prerender;
 class CORE_EXPORT PrerendererClient : public Supplement<Page> {
 public:
     virtual void willAddPrerender(Prerender*) = 0;
+    virtual bool isPrefetchOnly() = 0;
 
     static const char* supplementName();
     static PrerendererClient* from(Page*);

third_party/WebKit/Source/web/PrerendererClientImpl.cpp

@@ -51,4 +51,9 @@ void PrerendererClientImpl::willAddPrerender(Prerender* prerender)
     m_client->willAddPrerender(&webPrerender);
 }
 
+bool PrerendererClientImpl::isPrefetchOnly()
+{
+    return m_client && m_client->isPrefetchOnly();
+}
+
 } // namespace blink

third_party/WebKit/Source/web/PrerendererClientImpl.h

@@ -48,6 +48,7 @@ public:
     explicit PrerendererClientImpl(WebPrerendererClient*);
 
     void willAddPrerender(Prerender*) override;
+    bool isPrefetchOnly() override;
 
     DEFINE_INLINE_VIRTUAL_TRACE() { PrerendererClient::trace(visitor); }
 

third_party/WebKit/Source/web/tests/PrerenderingTest.cpp

@@ -98,6 +98,11 @@ private:
         m_webPrerenders.push_back(*prerender);
     }
 
+    bool isPrefetchOnly() override
+    {
+        return false;
+    }
+
     std::unique_ptr<WebPrerender::ExtraData> m_extraData;
     std::list<WebPrerender> m_webPrerenders;
 };

third_party/WebKit/public/web/WebPrerendererClient.h

@@ -41,6 +41,7 @@ class WebPrerender;
 class WebPrerendererClient {
 public:
     virtual void willAddPrerender(WebPrerender*) = 0;
+    virtual bool isPrefetchOnly() = 0;
 };
 
 } // namespace blink