Create untrusted offline page when opeing MHTML via file/content URLs

Bug: 758690
Tbr: ryansturm@chromium.org
Change-Id: Ie132c4806e7b59b00e0bc5e0adb72e6631ed3cf9
Reviewed-on: https://chromium-review.googlesource.com/817526
Commit-Queue: Jian Li <jianli@chromium.org>
Reviewed-by: Ryan Sturm <ryansturm@chromium.org>
Reviewed-by: Filip Gorski <fgorski@chromium.org>
Reviewed-by: Camille Lamy <clamy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#524490}

chrome/browser/offline_pages/offline_page_request_job.cc

@@ -377,8 +377,7 @@ RequestResult AccessOfflineFile(
       OfflinePageTabHelper::FromWebContents(web_contents);
   DCHECK(tab_helper);
   tab_helper->SetOfflinePage(
-      *offline_page,
-      offline_header,
+      *offline_page, offline_header, true /*is_trusted*/,
       network_state == NetworkState::PROHIBITIVELY_SLOW_NETWORK);
 
   *offline_file_path = offline_page->file_path;

chrome/browser/offline_pages/offline_page_tab_helper.cc

@@ -8,11 +8,13 @@
 #include "base/guid.h"
 #include "base/logging.h"
 #include "base/memory/ptr_util.h"
+#include "build/build_config.h"
 #include "chrome/browser/offline_pages/offline_page_request_job.h"
 #include "chrome/browser/offline_pages/prefetch/prefetch_service_factory.h"
 #include "chrome/browser/offline_pages/request_coordinator_factory.h"
 #include "components/offline_pages/core/background/request_coordinator.h"
 #include "components/offline_pages/core/offline_page_item.h"
+#include "components/offline_pages/core/offline_store_utils.h"
 #include "components/offline_pages/core/prefetch/offline_metrics_collector.h"
 #include "components/offline_pages/core/prefetch/prefetch_service.h"
 #include "content/public/browser/browser_thread.h"
@@ -27,14 +29,25 @@ DEFINE_WEB_CONTENTS_USER_DATA_KEY(offline_pages::OfflinePageTabHelper);
 
 namespace offline_pages {
 
+namespace {
+bool SchemeIsForUntrustedOfflinePages(const GURL& url) {
+#if defined(OS_ANDROID)
+  if (url.SchemeIs(url::kContentScheme))
+    return true;
+#endif
+  return url.SchemeIsFile();
+}
+}  // namespace
+
 OfflinePageTabHelper::LoadedOfflinePageInfo::LoadedOfflinePageInfo()
-    : is_showing_offline_preview(false) {}
+    : is_trusted(false), is_showing_offline_preview(false) {}
 
 OfflinePageTabHelper::LoadedOfflinePageInfo::~LoadedOfflinePageInfo() {}
 
 void OfflinePageTabHelper::LoadedOfflinePageInfo::Clear() {
   offline_page.reset();
   offline_header.Clear();
+  is_trusted = false;
   is_showing_offline_preview = false;
 }
 
@@ -87,52 +100,83 @@ void OfflinePageTabHelper::DidFinishNavigation(
   if (navigation_handle->IsSameDocument())
     return;
 
+  FinalizeOfflineInfo(navigation_handle);
+  provisional_offline_info_.Clear();
+
+  ReportPrefetchMetrics(navigation_handle);
+
+  TryLoadingOfflinePageOnNetError(navigation_handle);
+}
+
+void OfflinePageTabHelper::FinalizeOfflineInfo(
+    content::NavigationHandle* navigation_handle) {
+  offline_info_.Clear();
+
+  if (navigation_handle->IsErrorPage())
+    return;
+
   GURL navigated_url = navigation_handle->GetURL();
-  if (navigation_handle->IsErrorPage()) {
-    offline_info_.Clear();
-  } else {
-    // The provisional offline info can now be committed if the navigation is
-    // done without error.
-    DCHECK(!provisional_offline_info_.offline_page ||
-           OfflinePageUtils::EqualsIgnoringFragment(
-               navigated_url,
-               provisional_offline_info_.offline_page->url));
-    offline_info_.offline_page =
-        std::move(provisional_offline_info_.offline_page);
-    offline_info_.offline_header = provisional_offline_info_.offline_header;
-    offline_info_.is_showing_offline_preview =
-        provisional_offline_info_.is_showing_offline_preview;
-
-    // Report prefetch usage to OfflineMetricsCollector.
-    if (offline_info_.offline_page &&
-        policy_controller_.IsSuggested(
-            offline_info_.offline_page->client_id.name_space)) {
-      prefetch_service_->GetOfflineMetricsCollector()->OnPrefetchedPageOpened();
-    }
+
+  // If a MHTML archive is being loaded for file: or content: URL, create an
+  // untrusted offline page.
+  if (SchemeIsForUntrustedOfflinePages(navigated_url) &&
+      navigation_handle->GetWebContents()->GetContentsMimeType() ==
+          "multipart/related") {
+    offline_info_.offline_page = std::make_unique<OfflinePageItem>();
+    offline_info_.offline_page->offline_id = store_utils::GenerateOfflineId();
+    offline_info_.is_trusted = false;
+    // TODO(jianli): Extract the url where the MHTML acrhive claims from the
+    // MHTML headers and set it in OfflinePageItem::original_url.
+    return;
   }
-  provisional_offline_info_.Clear();
+
+  // For http/https URL,commit the provisional offline info if any.
+  if (!navigated_url.SchemeIsHTTPOrHTTPS() ||
+      !provisional_offline_info_.offline_page) {
+    return;
+  }
+
+  DCHECK(OfflinePageUtils::EqualsIgnoringFragment(
+      navigated_url, provisional_offline_info_.offline_page->url));
+  offline_info_.offline_page =
+      std::move(provisional_offline_info_.offline_page);
+  offline_info_.offline_header = provisional_offline_info_.offline_header;
+  offline_info_.is_trusted = provisional_offline_info_.is_trusted;
+  offline_info_.is_showing_offline_preview =
+      provisional_offline_info_.is_showing_offline_preview;
+}
+
+void OfflinePageTabHelper::ReportPrefetchMetrics(
+    content::NavigationHandle* navigation_handle) {
+  if (navigation_handle->IsErrorPage())
+    return;
+
+  if (!prefetch_service_)
+    return;
 
   // Report the kind of navigation (online/offline) to metrics collector.
   // It accumulates this info to mark a day as 'offline' or 'online'.
-  if (!navigation_handle->IsErrorPage()) {
-    if (prefetch_service_) {
-      OfflineMetricsCollector* metrics_collector =
-          prefetch_service_->GetOfflineMetricsCollector();
-      DCHECK(metrics_collector);
-
-      if (offline_page()) {
-        // Note that navigation to offline page may happen even if network is
-        // connected. For the purposes of collecting offline usage statistics,
-        // we still count this as offline navigation.
-        metrics_collector->OnSuccessfulNavigationOffline();
-      } else {
-        metrics_collector->OnSuccessfulNavigationOnline();
-        // The device is apparently online, attempt to report stats to UMA.
-        metrics_collector->ReportAccumulatedStats();
-      }
-    }
+  OfflineMetricsCollector* metrics_collector =
+      prefetch_service_->GetOfflineMetricsCollector();
+  DCHECK(metrics_collector);
+
+  if (offline_page()) {
+    // Report prefetch usage.
+    if (policy_controller_.IsSuggested(offline_page()->client_id.name_space))
+      metrics_collector->OnPrefetchedPageOpened();
+    // Note that navigation to offline page may happen even if network is
+    // connected. For the purposes of collecting offline usage statistics,
+    // we still count this as offline navigation.
+    metrics_collector->OnSuccessfulNavigationOffline();
+  } else {
+    metrics_collector->OnSuccessfulNavigationOnline();
+    // The device is apparently online, attempt to report stats to UMA.
+    metrics_collector->ReportAccumulatedStats();
   }
+}
 
+void OfflinePageTabHelper::TryLoadingOfflinePageOnNetError(
+    content::NavigationHandle* navigation_handle) {
   // If the offline page has been loaded successfully, nothing more to do.
   net::Error error_code = navigation_handle->GetNetErrorCode();
   if (error_code == net::OK)
@@ -174,7 +218,7 @@ void OfflinePageTabHelper::DidFinishNavigation(
   }
 
   OfflinePageUtils::SelectPageForURL(
-      web_contents()->GetBrowserContext(), navigated_url,
+      web_contents()->GetBrowserContext(), navigation_handle->GetURL(),
       URLSearchMode::SEARCH_BY_ALL_URLS, tab_id,
       base::Bind(&OfflinePageTabHelper::SelectPageForURLDone,
                  weak_ptr_factory_.GetWeakPtr()));
@@ -206,13 +250,19 @@ void OfflinePageTabHelper::SelectPageForURLDone(
 void OfflinePageTabHelper::SetOfflinePage(
     const OfflinePageItem& offline_page,
     const OfflinePageHeader& offline_header,
+    bool is_trusted,
     bool is_offline_preview) {
   provisional_offline_info_.offline_page =
       base::MakeUnique<OfflinePageItem>(offline_page);
   provisional_offline_info_.offline_header = offline_header;
+  provisional_offline_info_.is_trusted = is_trusted;
   provisional_offline_info_.is_showing_offline_preview = is_offline_preview;
 }
 
+bool OfflinePageTabHelper::IsShowingTrustedOfflinePage() const {
+  return offline_info_.offline_page && offline_info_.is_trusted;
+}
+
 const OfflinePageItem* OfflinePageTabHelper::GetOfflinePageForTest() const {
   return provisional_offline_info_.offline_page.get();
 }

chrome/browser/offline_pages/offline_page_tab_helper.h

@@ -33,6 +33,7 @@ class OfflinePageTabHelper :
 
   void SetOfflinePage(const OfflinePageItem& offline_page,
                       const OfflinePageHeader& offline_header,
+                      bool is_trusted,
                       bool is_offline_preview);
 
   const OfflinePageItem* offline_page() {
@@ -43,6 +44,9 @@ class OfflinePageTabHelper :
     return offline_info_.offline_header;
   }
 
+  // Returns whether a trusted offline page is being displayed.
+  bool IsShowingTrustedOfflinePage() const;
+
   // Returns nullptr if the page is not an offline preview. Returns the
   // OfflinePageItem related to the page if the page is an offline preview.
   const OfflinePageItem* GetOfflinePreviewItem() const;
@@ -78,6 +82,9 @@ class OfflinePageTabHelper :
     // The offline header that is provided when offline page is loaded.
     OfflinePageHeader offline_header;
 
+    // Whether the page is deemed trusted or not.
+    bool is_trusted;
+
     // Whether the page is an offline preview. Offline page previews are shown
     // when a user's effective connection type is prohibitively slow.
     bool is_showing_offline_preview;
@@ -93,6 +100,16 @@ class OfflinePageTabHelper :
   void DidFinishNavigation(
       content::NavigationHandle* navigation_handle) override;
 
+  // Finalize the offline info when the navigation is done.
+  void FinalizeOfflineInfo(content::NavigationHandle* navigation_handle);
+
+  // Report the metrics essential to PrefetchService.
+  void ReportPrefetchMetrics(content::NavigationHandle* navigation_handle);
+
+  // Reload the URL in order to fetch the offline page on certain net errors.
+  void TryLoadingOfflinePageOnNetError(
+      content::NavigationHandle* navigation_handle);
+
   void SelectPageForURLDone(const OfflinePageItem* offline_page);
 
   void DuplicateCheckDoneForScheduleDownload(

chrome/browser/offline_pages/offline_page_tab_helper_unittest.cc

@@ -8,6 +8,7 @@
 
 #include "base/memory/ptr_util.h"
 #include "base/threading/thread_task_runner_handle.h"
+#include "build/build_config.h"
 #include "chrome/browser/offline_pages/prefetch/prefetch_service_factory.h"
 #include "chrome/test/base/testing_profile.h"
 #include "components/offline_pages/core/offline_page_item.h"
@@ -22,6 +23,13 @@
 
 namespace {
 const GURL kTestPageUrl("http://mystery.site/foo.html");
+const GURL kTestFileUrl("file://foo");
+
+#if defined(OS_ANDROID)
+const GURL kTestContentUrl("content://foo");
+#endif
+
+const char kTestHeader[] = "reason=download";
 }  // namespace
 
 namespace offline_pages {
@@ -68,6 +76,8 @@ class OfflinePageTabHelperTest : public content::RenderViewHostTestHarness {
   void TearDown() override;
   content::BrowserContext* CreateBrowserContext() override;
 
+  void CreateNavigationSimulator(const GURL& url);
+
   OfflinePageTabHelper* tab_helper() const { return tab_helper_; }
   PrefetchService* prefetch_service() const { return prefetch_service_; }
   content::NavigationSimulator* navigation_simulator() {
@@ -98,15 +108,8 @@ void OfflinePageTabHelperTest::SetUp() {
   prefetch_service_ =
       PrefetchServiceFactory::GetForBrowserContext(browser_context());
 
-  // This initializes a nav stack inside the harness.
-  NavigateAndCommit(kTestPageUrl);
-
   OfflinePageTabHelper::CreateForWebContents(web_contents());
   tab_helper_ = OfflinePageTabHelper::FromWebContents(web_contents());
-
-  navigation_simulator_ = content::NavigationSimulator::CreateRendererInitiated(
-      kTestPageUrl, main_rfh());
-  navigation_simulator_->SetTransition(ui::PAGE_TRANSITION_LINK);
 }
 
 void OfflinePageTabHelperTest::TearDown() {
@@ -118,8 +121,15 @@ content::BrowserContext* OfflinePageTabHelperTest::CreateBrowserContext() {
   return builder.Build().release();
 }
 
+void OfflinePageTabHelperTest::CreateNavigationSimulator(const GURL& url) {
+  navigation_simulator_ =
+      content::NavigationSimulator::CreateBrowserInitiated(url, web_contents());
+  navigation_simulator_->SetTransition(ui::PAGE_TRANSITION_LINK);
+}
+
 // Checks the test setup.
 TEST_F(OfflinePageTabHelperTest, InitialSetup) {
+  CreateNavigationSimulator(kTestPageUrl);
   EXPECT_NE(nullptr, tab_helper());
   EXPECT_NE(nullptr, prefetch_service());
   EXPECT_NE(nullptr, prefetch_service()->GetOfflineMetricsCollector());
@@ -131,6 +141,7 @@ TEST_F(OfflinePageTabHelperTest, InitialSetup) {
 }
 
 TEST_F(OfflinePageTabHelperTest, MetricsStartNavigation) {
+  CreateNavigationSimulator(kTestPageUrl);
   // This causes WCO::DidStartNavigation()
   navigation_simulator()->Start();
 
@@ -141,6 +152,7 @@ TEST_F(OfflinePageTabHelperTest, MetricsStartNavigation) {
 }
 
 TEST_F(OfflinePageTabHelperTest, MetricsOnlineNavigation) {
+  CreateNavigationSimulator(kTestPageUrl);
   navigation_simulator()->Start();
   navigation_simulator()->Commit();
 
@@ -152,12 +164,13 @@ TEST_F(OfflinePageTabHelperTest, MetricsOnlineNavigation) {
 }
 
 TEST_F(OfflinePageTabHelperTest, MetricsOfflineNavigation) {
+  CreateNavigationSimulator(kTestPageUrl);
   navigation_simulator()->Start();
 
   // Simulate offline interceptor loading an offline page instead.
   OfflinePageItem offlinePage(kTestPageUrl, 0, ClientId(), base::FilePath(), 0);
   OfflinePageHeader offlineHeader;
-  tab_helper()->SetOfflinePage(offlinePage, offlineHeader, false);
+  tab_helper()->SetOfflinePage(offlinePage, offlineHeader, true, false);
 
   navigation_simulator()->Commit();
 
@@ -168,4 +181,47 @@ TEST_F(OfflinePageTabHelperTest, MetricsOfflineNavigation) {
   EXPECT_EQ(0, metrics()->report_stats_count_);
 }
 
+TEST_F(OfflinePageTabHelperTest, TrustedOfflinePage) {
+  CreateNavigationSimulator(kTestPageUrl);
+  navigation_simulator()->Start();
+
+  OfflinePageItem offlinePage(kTestPageUrl, 0, ClientId(), base::FilePath(), 0);
+  OfflinePageHeader offlineHeader(kTestHeader);
+  tab_helper()->SetOfflinePage(offlinePage, offlineHeader, true, false);
+
+  navigation_simulator()->Commit();
+
+  ASSERT_NE(nullptr, tab_helper()->offline_page());
+  EXPECT_EQ(kTestPageUrl, tab_helper()->offline_page()->url);
+  EXPECT_TRUE(tab_helper()->IsShowingTrustedOfflinePage());
+  EXPECT_EQ(OfflinePageHeader::Reason::DOWNLOAD,
+            tab_helper()->offline_header().reason);
+}
+
+TEST_F(OfflinePageTabHelperTest, UntrustedOfflinePageForFileUrl) {
+  CreateNavigationSimulator(kTestFileUrl);
+  navigation_simulator()->Start();
+  navigation_simulator()->SetContentsMimeType("multipart/related");
+  navigation_simulator()->Commit();
+
+  ASSERT_NE(nullptr, tab_helper()->offline_page());
+  EXPECT_FALSE(tab_helper()->IsShowingTrustedOfflinePage());
+  EXPECT_EQ(OfflinePageHeader::Reason::NONE,
+            tab_helper()->offline_header().reason);
+}
+
+#if defined(OS_ANDROID)
+TEST_F(OfflinePageTabHelperTest, UntrustedOfflinePageForContentUrl) {
+  CreateNavigationSimulator(kTestContentUrl);
+  navigation_simulator()->Start();
+  navigation_simulator()->SetContentsMimeType("multipart/related");
+  navigation_simulator()->Commit();
+
+  ASSERT_NE(nullptr, tab_helper()->offline_page());
+  EXPECT_FALSE(tab_helper()->IsShowingTrustedOfflinePage());
+  EXPECT_EQ(OfflinePageHeader::Reason::NONE,
+            tab_helper()->offline_header().reason);
+}
+#endif
+
 }  // namespace offline_pages

chrome/browser/offline_pages/offline_page_utils.cc

@@ -177,17 +177,21 @@ const OfflinePageItem* OfflinePageUtils::GetOfflinePageFromWebContents(
   const OfflinePageItem* offline_page = tab_helper->offline_page();
   if (!offline_page)
     return nullptr;
+  // TODO(jianli): Remove this when the UI knows how to handle untrusted
+  // offline pages.
+  if (!tab_helper->IsShowingTrustedOfflinePage())
+    return nullptr;
 
-  // Returns the cached offline page only if the offline URL matches with
-  // current tab URL (skipping fragment identifier part). This is to prevent
+  // If a pending navigation that hasn't committed yet, don't return the cached
+  // offline page that was set at the last commit time. This is to prevent
   // from returning the wrong offline page if DidStartNavigation is never called
   // to clear it up.
-  GURL::Replacements remove_params;
-  remove_params.ClearRef();
-  GURL offline_url = offline_page->url.ReplaceComponents(remove_params);
-  GURL web_contents_url =
-      web_contents->GetVisibleURL().ReplaceComponents(remove_params);
-  return offline_url == web_contents_url ? offline_page : nullptr;
+  if (!EqualsIgnoringFragment(web_contents->GetVisibleURL(),
+                              web_contents->GetLastCommittedURL())) {
+    return nullptr;
+  }
+
+  return offline_page;
 }
 
 // static

chrome/browser/previews/previews_infobar_tab_helper_unittest.cc

@@ -240,7 +240,7 @@ TEST_F(PreviewsInfoBarTabHelperUnitTest, CreateOfflineInfoBar) {
   int64_t expected_file_size = .55 * item.file_size;
   offline_pages::OfflinePageHeader header;
   offline_pages::OfflinePageTabHelper::FromWebContents(web_contents())
-      ->SetOfflinePage(item, header, true);
+      ->SetOfflinePage(item, header, true, true);
 
   auto* data_reduction_proxy_settings =
       DataReductionProxyChromeSettingsFactory::GetForBrowserContext(

components/offline_pages/core/offline_page_item.cc

@@ -81,7 +81,7 @@ bool OfflinePageItem::operator==(const OfflinePageItem& other) const {
          request_origin == other.request_origin &&
          system_download_id == other.system_download_id &&
          file_missing_time == other.file_missing_time &&
-         upgrade_attempt == other.upgrade_attempt && digest == digest;
+         upgrade_attempt == other.upgrade_attempt && digest == other.digest;
 }
 
 bool OfflinePageItem::operator<(const OfflinePageItem& other) const {

content/public/test/navigation_simulator.cc

@@ -237,6 +237,7 @@ NavigationSimulator::NavigationSimulator(const GURL& original_url,
       browser_initiated_(browser_initiated),
       transition_(browser_initiated ? ui::PAGE_TRANSITION_TYPED
                                     : ui::PAGE_TRANSITION_LINK),
+      contents_mime_type_("text/html"),
       weak_factory_(this) {
   // For renderer-initiated navigation, the RenderFrame must be initialized. Do
   // it if it hasn't happened yet.
@@ -471,7 +472,7 @@ void NavigationSimulator::Commit() {
   params.did_create_new_entry = DidCreateNewEntry();
   params.gesture =
       has_user_gesture_ ? NavigationGestureUser : NavigationGestureAuto;
-  params.contents_mime_type = "text/html";
+  params.contents_mime_type = contents_mime_type_;
   params.method = "GET";
   params.http_status_code = 200;
   params.history_list_was_cleared = false;
@@ -641,7 +642,7 @@ void NavigationSimulator::CommitSameDocument() {
   params.did_create_new_entry = false;
   params.gesture =
       has_user_gesture_ ? NavigationGestureUser : NavigationGestureAuto;
-  params.contents_mime_type = "text/html";
+  params.contents_mime_type = contents_mime_type_;
   params.method = "GET";
   params.http_status_code = 200;
   params.history_list_was_cleared = false;
@@ -713,6 +714,13 @@ void NavigationSimulator::SetInterfaceProviderRequest(
   interface_provider_request_ = std::move(request);
 }
 
+void NavigationSimulator::SetContentsMimeType(
+    const std::string& contents_mime_type) {
+  CHECK_LE(state_, STARTED) << "The contents mime type cannot be set after the "
+                               "navigation has committed or failed";
+  contents_mime_type_ = contents_mime_type;
+}
+
 NavigationThrottle::ThrottleCheckResult
 NavigationSimulator::GetLastThrottleCheckResult() {
   return last_throttle_check_result_.value();

content/public/test/navigation_simulator.h

@@ -233,6 +233,10 @@ class NavigationSimulator : public WebContentsObserver {
   virtual void SetInterfaceProviderRequest(
       service_manager::mojom::InterfaceProviderRequest request);
 
+  // Provides the contents mime type to be set at commit. It should be
+  // specified before calling |Commit|.
+  virtual void SetContentsMimeType(const std::string& contents_mime_type);
+
   // --------------------------------------------------------------------------
 
   // Gets the last throttle check result computed by the navigation throttles.
@@ -352,6 +356,7 @@ class NavigationSimulator : public WebContentsObserver {
   int session_history_offset_ = 0;
   bool has_user_gesture_ = true;
   service_manager::mojom::InterfaceProviderRequest interface_provider_request_;
+  std::string contents_mime_type_;
 
   // These are used to sanity check the content/public/ API calls emitted as
   // part of the navigation.