[Explore Sites] Adding URL fetcher for Explore Sites.

This will be used to fetch the data from our API endpoint. It is only a fetcher
and does not do any processing of the data that is downloaded.

Bug: 867488
Change-Id: Ic7506af2523632c1173333ddf2a1f2183b27f72f
Reviewed-on: https://chromium-review.googlesource.com/1229013
Commit-Queue: Jonathan Freed <freedjm@chromium.org>
Reviewed-by: Justin DeWitt <dewittj@chromium.org>
Cr-Commit-Position: refs/heads/master@{#592919}

chrome/browser/BUILD.gn

@@ -2136,6 +2136,8 @@ jumbo_split_static_library("browser") {
       "android/explore_sites/explore_sites_bridge_experimental.cc",
       "android/explore_sites/explore_sites_feature.cc",
       "android/explore_sites/explore_sites_feature.h",
+      "android/explore_sites/explore_sites_fetcher.cc",
+      "android/explore_sites/explore_sites_fetcher.h",
       "android/explore_sites/explore_sites_schema.cc",
       "android/explore_sites/explore_sites_schema.h",
       "android/explore_sites/explore_sites_service.h",
@@ -2157,6 +2159,8 @@ jumbo_split_static_library("browser") {
       "android/explore_sites/ntp_json_fetcher.h",
       "android/explore_sites/url_util.cc",
       "android/explore_sites/url_util.h",
+      "android/explore_sites/url_util_experimental.cc",
+      "android/explore_sites/url_util_experimental.h",
       "android/favicon_helper.cc",
       "android/favicon_helper.h",
       "android/feature_engagement/tracker_factory_android.cc",

chrome/browser/android/explore_sites/catalog.proto

@@ -75,3 +75,23 @@ message Site {
   // language preferences.
   string title = 3;
 }
+
+message GetCatalogRequest {
+  // The latest timestamp of the catalog on the client.
+  int64 created_at_millis = 1;
+
+  // The country code for the catalog that was returned.
+  string country_code = 2;
+}
+
+message GetCatalogResponse {
+  // Catalog of categories and sites that are appropriate for the client. Will
+  // be empty if the client sends a request with the latest timestamp.
+  Catalog catalog = 1;
+
+  // The latest timestamp of the catalof for the client on the server.
+  int64 created_at_millis = 2;
+
+  // The country code for the catalog that was returned.
+  string country_code = 3;
+}
\ No newline at end of file

chrome/browser/android/explore_sites/explore_sites_fetcher.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/android/explore_sites/explore_sites_fetcher.h"
+
+#include <utility>
+
+#include "base/bind.h"
+#include "base/logging.h"
+#include "base/memory/ptr_util.h"
+#include "base/metrics/field_trial_params.h"
+#include "base/strings/stringprintf.h"
+#include "base/values.h"
+#include "base/version.h"
+#include "chrome/browser/android/chrome_feature_list.h"
+#include "chrome/browser/android/explore_sites/catalog.pb.h"
+#include "chrome/browser/android/explore_sites/explore_sites_types.h"
+#include "chrome/browser/android/explore_sites/url_util.h"
+#include "chrome/browser/net/chrome_accept_language_settings.h"
+#include "chrome/common/channel_info.h"
+#include "components/version_info/version_info.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/storage_partition.h"
+#include "content/public/common/service_manager_connection.h"
+#include "google_apis/google_api_keys.h"
+#include "net/base/load_flags.h"
+#include "net/base/url_util.h"
+#include "net/http/http_status_code.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_request_status.h"
+#include "services/network/public/cpp/resource_request.h"
+#include "services/network/public/cpp/simple_url_loader.h"
+#include "url/gurl.h"
+
+namespace explore_sites {
+
+namespace {
+
+// Content type needed in order to communicate with the server in binary
+// proto format.
+const char kRequestContentType[] = "application/x-protobuf";
+const char kRequestMethod[] = "POST";
+
+constexpr net::NetworkTrafficAnnotationTag traffic_annotation =
+    net::DefineNetworkTrafficAnnotation("explore_sites", R"(
+          semantics {
+            sender: "Explore Sites"
+            description:
+              "Downloads a catalog of categories and sites for the purposes of "
+              "exploring the Web."
+            trigger:
+              "Periodically scheduled for update in the background and also "
+              "triggered by New Tab Page UI."
+            data:
+              "Proto data comprising interesting site and category information."
+              " No user information is sent."
+            destination: GOOGLE_OWNED_SERVICE
+          }
+          policy {
+            cookies_allowed: NO
+          })");
+
+}  // namespace
+
+std::unique_ptr<ExploreSitesFetcher> ExploreSitesFetcher::CreateForGetCatalog(
+    Callback callback,
+    const int64_t catalog_version,
+    const std::string country_code,
+    scoped_refptr<network::SharedURLLoaderFactory> loader_factory) {
+  GURL url = GetCatalogURL();
+  return base::WrapUnique(new ExploreSitesFetcher(
+      std::move(callback), url, catalog_version, country_code, loader_factory));
+}
+
+std::unique_ptr<ExploreSitesFetcher>
+ExploreSitesFetcher::CreateForGetCategories(
+    Callback callback,
+    const int64_t catalog_version,
+    const std::string country_code,
+    scoped_refptr<network::SharedURLLoaderFactory> loader_factory) {
+  GURL url = GetCategoriesURL();
+  return base::WrapUnique(new ExploreSitesFetcher(
+      std::move(callback), url, catalog_version, country_code, loader_factory));
+}
+
+ExploreSitesFetcher::ExploreSitesFetcher(
+    Callback callback,
+    const GURL& url,
+    const int64_t catalog_version,
+    const std::string country_code,
+    scoped_refptr<network::SharedURLLoaderFactory> loader_factory)
+    : callback_(std::move(callback)),
+      url_loader_factory_(loader_factory),
+      weak_factory_(this) {
+  base::Version version = version_info::GetVersion();
+  std::string channel_name = chrome::GetChannelName();
+  std::string client_version =
+      base::StringPrintf("%d.%d.%d.%s.chrome",
+                         version.components()[0],  // Major
+                         version.components()[2],  // Build
+                         version.components()[3],  // Patch
+                         channel_name.c_str());
+  auto resource_request = std::make_unique<network::ResourceRequest>();
+  resource_request->method = kRequestMethod;
+  bool is_stable_channel =
+      chrome::GetChannel() == version_info::Channel::STABLE;
+  std::string api_key = is_stable_channel ? google_apis::GetAPIKey()
+                                          : google_apis::GetNonStableAPIKey();
+  resource_request->headers.SetHeader("x-google-api-key", api_key);
+  resource_request->headers.SetHeader("X-Client-Version", client_version);
+  // TODO(freedjm): Implement Accept-Language support.
+
+  url_loader_ = network::SimpleURLLoader::Create(std::move(resource_request),
+                                                 traffic_annotation);
+
+  GetCatalogRequest request;
+  request.set_created_at_millis(catalog_version);
+  request.set_country_code(country_code);
+  std::string request_message;
+  request.SerializeToString(&request_message);
+  url_loader_->AttachStringForUpload(request_message, kRequestContentType);
+
+  url_loader_->DownloadToStringOfUnboundedSizeUntilCrashAndDie(
+      url_loader_factory_.get(),
+      base::BindOnce(&ExploreSitesFetcher::OnSimpleLoaderComplete,
+                     weak_factory_.GetWeakPtr()));
+}
+
+ExploreSitesFetcher::~ExploreSitesFetcher() {}
+
+void ExploreSitesFetcher::OnSimpleLoaderComplete(
+    std::unique_ptr<std::string> response_body) {
+  ExploreSitesRequestStatus status = HandleResponseCode();
+
+  if (response_body && response_body->empty()) {
+    DVLOG(1) << "Failed to get response or empty response";
+    status = ExploreSitesRequestStatus::kShouldRetry;
+  }
+
+  std::move(callback_).Run(status, std::move(response_body));
+}
+
+ExploreSitesRequestStatus ExploreSitesFetcher::HandleResponseCode() {
+  int response_code = -1;
+  if (url_loader_->ResponseInfo() && url_loader_->ResponseInfo()->headers)
+    response_code = url_loader_->ResponseInfo()->headers->response_code();
+
+  if (response_code == -1) {
+    int net_error = url_loader_->NetError();
+    DVLOG(1) << "Net error: " << net_error;
+    return (net_error == net::ERR_BLOCKED_BY_ADMINISTRATOR)
+               ? ExploreSitesRequestStatus::kShouldSuspendBlockedByAdministrator
+               : ExploreSitesRequestStatus::kShouldRetry;
+  } else if (response_code < 200 || response_code > 299) {
+    DVLOG(1) << "HTTP status: " << response_code;
+    switch (response_code) {
+      case net::HTTP_BAD_REQUEST:
+        return ExploreSitesRequestStatus::kShouldSuspendBadRequest;
+      default:
+        return ExploreSitesRequestStatus::kShouldRetry;
+    }
+  }
+
+  return ExploreSitesRequestStatus::kSuccess;
+}
+
+}  // namespace explore_sites

chrome/browser/android/explore_sites/explore_sites_fetcher.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_ANDROID_EXPLORE_SITES_EXPLORE_SITES_FETCHER_H_
+#define CHROME_BROWSER_ANDROID_EXPLORE_SITES_EXPLORE_SITES_FETCHER_H_
+
+#include <memory>
+#include <string>
+
+#include "base/callback.h"
+#include "base/macros.h"
+#include "base/memory/weak_ptr.h"
+#include "chrome/browser/android/explore_sites/explore_sites_types.h"
+
+namespace network {
+class SimpleURLLoader;
+class SharedURLLoaderFactory;
+}  // namespace network
+
+namespace explore_sites {
+
+// A class that fetches data from the server.
+class ExploreSitesFetcher {
+ public:
+  // Callback to pass back the catalog returned from the server.
+  // Invoked with |nullptr| if there is an error.
+  using Callback =
+      base::OnceCallback<void(ExploreSitesRequestStatus status,
+                              const std::unique_ptr<std::string> data)>;
+
+  // Creates a fetcher for the GetCatalog RPC.
+  static std::unique_ptr<ExploreSitesFetcher> CreateForGetCatalog(
+      Callback callback,
+      const int64_t catalog_version,
+      const std::string country_code,
+      scoped_refptr<network::SharedURLLoaderFactory> loader_factory);
+
+  // Creates a fetcher for the GetCategories RPC.
+  static std::unique_ptr<ExploreSitesFetcher> CreateForGetCategories(
+      Callback callback,
+      const int64_t catalog_version,
+      const std::string country_code,
+      scoped_refptr<network::SharedURLLoaderFactory> loader_factory);
+
+  ~ExploreSitesFetcher();
+
+ private:
+  explicit ExploreSitesFetcher(
+      Callback callback,
+      const GURL& url,
+      const int64_t catalog_version,
+      const std::string country_code,
+      scoped_refptr<network ::SharedURLLoaderFactory> loader_factory);
+
+  // Invoked from SimpleURLLoader after download is complete.
+  void OnSimpleLoaderComplete(std::unique_ptr<std::string> response_body);
+
+  ExploreSitesRequestStatus HandleResponseCode();
+
+  Callback callback_;
+  scoped_refptr<network::SharedURLLoaderFactory> url_loader_factory_;
+  std::unique_ptr<network::SimpleURLLoader> url_loader_;
+  base::WeakPtrFactory<ExploreSitesFetcher> weak_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(ExploreSitesFetcher);
+};
+
+}  // namespace explore_sites
+
+#endif  // CHROME_BROWSER_ANDROID_EXPLORE_SITES_EXPLORE_SITES_FETCHER_H_

chrome/browser/android/explore_sites/explore_sites_fetcher_unittest.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/android/explore_sites/explore_sites_fetcher.h"
+
+#include "base/message_loop/message_loop.h"
+#include "base/test/bind_test_util.h"
+#include "base/test/mock_callback.h"
+#include "base/test/test_mock_time_task_runner.h"
+#include "chrome/browser/android/explore_sites/catalog.pb.h"
+#include "chrome/browser/android/explore_sites/explore_sites_types.h"
+#include "net/http/http_status_code.h"
+#include "net/url_request/url_request_status.h"
+#include "services/network/public/cpp/weak_wrapper_shared_url_loader_factory.h"
+#include "services/network/test/test_url_loader_factory.h"
+#include "services/network/test/test_utils.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "url/gurl.h"
+
+using testing::_;
+using testing::DoAll;
+using testing::Eq;
+using testing::SaveArg;
+
+namespace explore_sites {
+
+// TODO(freedjm): Add tests for the headers.
+class ExploreSitesFetcherTest : public testing::Test {
+ public:
+  ExploreSitesFetcherTest();
+
+  void SetUp() override;
+
+  ExploreSitesRequestStatus RunFetcherWithNetError(net::Error net_error);
+  ExploreSitesRequestStatus RunFetcherWithHttpError(
+      net::HttpStatusCode http_error);
+  ExploreSitesRequestStatus RunFetcherWithData(const std::string& response_data,
+                                               std::string* data_received);
+
+ private:
+  ExploreSitesFetcher::Callback StoreResult();
+  network::TestURLLoaderFactory::PendingRequest* GetPendingRequest(
+      size_t index);
+  ExploreSitesRequestStatus RunFetcher(
+      base::OnceCallback<void(void)> respond_callback,
+      std::string* data_received);
+  void RespondWithData(const std::string& data);
+  void RespondWithNetError(int net_error);
+  void RespondWithHttpError(net::HttpStatusCode http_error);
+
+  network::TestURLLoaderFactory test_url_loader_factory_;
+  scoped_refptr<network::SharedURLLoaderFactory>
+      test_shared_url_loader_factory_;
+  network::ResourceRequest last_resource_request_;
+
+  ExploreSitesRequestStatus last_status_;
+  std::unique_ptr<std::string> last_data_;
+  base::MessageLoopForIO message_loop_;
+  scoped_refptr<base::TestMockTimeTaskRunner> task_runner_;
+};
+
+ExploreSitesFetcher::Callback ExploreSitesFetcherTest::StoreResult() {
+  return base::BindLambdaForTesting(
+      [&](ExploreSitesRequestStatus status, std::unique_ptr<std::string> data) {
+        last_status_ = status;
+        last_data_ = std::move(data);
+      });
+}
+
+ExploreSitesFetcherTest::ExploreSitesFetcherTest()
+    : test_shared_url_loader_factory_(
+          base::MakeRefCounted<network::WeakWrapperSharedURLLoaderFactory>(
+              &test_url_loader_factory_)),
+      task_runner_(new base::TestMockTimeTaskRunner) {
+  message_loop_.SetTaskRunner(task_runner_);
+}
+
+void ExploreSitesFetcherTest::SetUp() {
+  test_url_loader_factory_.SetInterceptor(
+      base::BindLambdaForTesting([&](const network::ResourceRequest& request) {
+        last_resource_request_ = request;
+      }));
+}
+
+ExploreSitesRequestStatus ExploreSitesFetcherTest::RunFetcherWithNetError(
+    net::Error net_error) {
+  std::string data_received;
+  ExploreSitesRequestStatus status =
+      RunFetcher(base::BindOnce(&ExploreSitesFetcherTest::RespondWithNetError,
+                                base::Unretained(this), net_error),
+                 &data_received);
+  EXPECT_TRUE(data_received.empty());
+  return status;
+}
+
+ExploreSitesRequestStatus ExploreSitesFetcherTest::RunFetcherWithHttpError(
+    net::HttpStatusCode http_error) {
+  std::string data_received;
+  ExploreSitesRequestStatus status =
+      RunFetcher(base::BindOnce(&ExploreSitesFetcherTest::RespondWithHttpError,
+                                base::Unretained(this), http_error),
+                 &data_received);
+  EXPECT_TRUE(data_received.empty());
+  return status;
+}
+
+ExploreSitesRequestStatus ExploreSitesFetcherTest::RunFetcherWithData(
+    const std::string& response_data,
+    std::string* data_received) {
+  return RunFetcher(base::BindOnce(&ExploreSitesFetcherTest::RespondWithData,
+                                   base::Unretained(this), response_data),
+                    data_received);
+}
+
+void ExploreSitesFetcherTest::RespondWithNetError(int net_error) {
+  int pending_requests_count = test_url_loader_factory_.NumPending();
+  DCHECK(pending_requests_count > 0);
+  network::URLLoaderCompletionStatus completion_status(net_error);
+  test_url_loader_factory_.SimulateResponseForPendingRequest(
+      GetPendingRequest(0)->request.url, completion_status,
+      network::ResourceResponseHead(), std::string());
+}
+
+void ExploreSitesFetcherTest::RespondWithHttpError(
+    net::HttpStatusCode http_error) {
+  int pending_requests_count = test_url_loader_factory_.NumPending();
+  auto resource_response_head = network::CreateResourceResponseHead(http_error);
+  DCHECK(pending_requests_count > 0);
+  test_url_loader_factory_.SimulateResponseForPendingRequest(
+      GetPendingRequest(0)->request.url,
+      network::URLLoaderCompletionStatus(net::OK), resource_response_head,
+      std::string());
+}
+
+void ExploreSitesFetcherTest::RespondWithData(const std::string& data) {
+  DCHECK(test_url_loader_factory_.pending_requests()->size() > 0);
+  test_url_loader_factory_.SimulateResponseForPendingRequest(
+      GetPendingRequest(0)->request.url.spec(), data);
+}
+
+network::TestURLLoaderFactory::PendingRequest*
+ExploreSitesFetcherTest::GetPendingRequest(size_t index) {
+  if (index >= test_url_loader_factory_.pending_requests()->size())
+    return nullptr;
+  network::TestURLLoaderFactory::PendingRequest* request =
+      &(*test_url_loader_factory_.pending_requests())[index];
+  DCHECK(request);
+  return request;
+}
+
+ExploreSitesRequestStatus ExploreSitesFetcherTest::RunFetcher(
+    base::OnceCallback<void(void)> respond_callback,
+    std::string* data_received) {
+  std::unique_ptr<ExploreSitesFetcher> fetcher =
+      ExploreSitesFetcher::CreateForGetCatalog(StoreResult(), 0, "KE",
+                                               test_shared_url_loader_factory_);
+
+  std::move(respond_callback).Run();
+  task_runner_->RunUntilIdle();
+
+  if (last_data_)
+    *data_received = *last_data_;
+  return last_status_;
+}
+
+TEST_F(ExploreSitesFetcherTest, NetErrors) {
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldSuspendBlockedByAdministrator,
+            RunFetcherWithNetError(net::ERR_BLOCKED_BY_ADMINISTRATOR));
+
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithNetError(net::ERR_INTERNET_DISCONNECTED));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithNetError(net::ERR_NETWORK_CHANGED));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithNetError(net::ERR_CONNECTION_RESET));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithNetError(net::ERR_CONNECTION_CLOSED));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithNetError(net::ERR_CONNECTION_REFUSED));
+}
+
+TEST_F(ExploreSitesFetcherTest, HttpErrors) {
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldSuspendBadRequest,
+            RunFetcherWithHttpError(net::HTTP_BAD_REQUEST));
+
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_NOT_IMPLEMENTED));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_UNAUTHORIZED));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_NOT_FOUND));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_CONFLICT));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_INTERNAL_SERVER_ERROR));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_BAD_GATEWAY));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_SERVICE_UNAVAILABLE));
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithHttpError(net::HTTP_GATEWAY_TIMEOUT));
+}
+
+TEST_F(ExploreSitesFetcherTest, EmptyResponse) {
+  std::string data;
+  EXPECT_EQ(ExploreSitesRequestStatus::kShouldRetry,
+            RunFetcherWithData("", &data));
+  EXPECT_TRUE(data.empty());
+}
+
+TEST_F(ExploreSitesFetcherTest, Success) {
+  std::string data;
+  EXPECT_EQ(ExploreSitesRequestStatus::kSuccess,
+            RunFetcherWithData("Any data.", &data));
+  EXPECT_FALSE(data.empty());
+  EXPECT_EQ(data, "Any data.");
+}
+
+}  // namespace explore_sites

chrome/browser/android/explore_sites/explore_sites_types.h

@@ -61,6 +61,21 @@ using EncodedImageList = std::vector<std::unique_ptr<EncodedImageBytes>>;
 using EncodedImageListCallback = base::OnceCallback<void(EncodedImageList)>;
 
 using BitmapCallback = base::OnceCallback<void(std::unique_ptr<SkBitmap>)>;
-}  // namespace explore_sites
 
+// Status for sending request to the server.
+enum class ExploreSitesRequestStatus {
+  // Request completed successfully.
+  kSuccess = 0,
+  // Request failed due to to local network problem, caller should retry.
+  kShouldRetry = 1,
+  // Request failed with error indicating that the request can not be serviced
+  // by the server.
+  kShouldSuspendBadRequest = 2,
+  // The request was blocked by a URL blacklist configured by the domain
+  // administrator.
+  kShouldSuspendBlockedByAdministrator = 3,
+  // kMaxValue should always be the last type.
+  kMaxValue = kShouldSuspendBlockedByAdministrator
+};
+}  // namespace explore_sites
 #endif  // CHROME_BROWSER_ANDROID_EXPLORE_SITES_EXPLORE_SITES_TYPES_H_

chrome/browser/android/explore_sites/ntp_json_fetcher.cc

@@ -12,7 +12,7 @@
 #include "base/metrics/field_trial_params.h"
 #include "base/values.h"
 #include "chrome/browser/android/chrome_feature_list.h"
-#include "chrome/browser/android/explore_sites/url_util.h"
+#include "chrome/browser/android/explore_sites/url_util_experimental.h"
 #include "content/public/browser/browser_context.h"
 #include "content/public/browser/storage_partition.h"
 #include "content/public/common/service_manager_connection.h"
@@ -66,7 +66,7 @@ void NTPJsonFetcher::Start(Callback callback) {
               "This feature is only enabled explicitly by flag."
           })");
   auto resource_request = std::make_unique<network::ResourceRequest>();
-  resource_request->url = GetNtpURL();
+  resource_request->url = GetNtpPrototypeURL();
   simple_loader_ = network::SimpleURLLoader::Create(std::move(resource_request),
                                                     traffic_annotation);
   network::mojom::URLLoaderFactory* loader_factory =
@@ -119,7 +119,7 @@ void NTPJsonFetcher::OnJsonParseSuccess(
 }
 
 void NTPJsonFetcher::OnJsonParseError(const std::string& error) {
-  DVLOG(1) << "Unable to parse NTP JSON from " << GetNtpURL()
+  DVLOG(1) << "Unable to parse NTP JSON from " << GetNtpPrototypeURL()
            << " error: " << error;
   std::move(callback_).Run(nullptr);
 }

chrome/browser/android/explore_sites/url_util.cc

@@ -14,8 +14,7 @@ namespace explore_sites {
 
 GURL GetBaseURL() {
   const char kBaseURLOption[] = "base_url";
-  const char kDefaultBaseUrl[] =
-      "https://explore-sites-ux-research.appspot.com";
+  const char kDefaultBaseUrl[] = "https://exploresites-pa.googleapis.com/v1";
   std::string field_trial_param = base::GetFieldTrialParamValueByFeature(
       chrome::android::kExploreSites, kBaseURLOption);
   if (field_trial_param.empty())
@@ -23,9 +22,9 @@ GURL GetBaseURL() {
   return GURL(field_trial_param);
 }
 
-GURL GetNtpURL() {
-  const char kNtpJsonPath[] = "/ntp.json";
-  std::string path(kNtpJsonPath);
+GURL GetCatalogURL() {
+  const char kGetCatalogPath[] = "/getcatalog";
+  std::string path(kGetCatalogPath);
 
   GURL base_url(GetBaseURL());
   GURL::Replacements replacements;
@@ -33,9 +32,9 @@ GURL GetNtpURL() {
   return base_url.ReplaceComponents(replacements);
 }
 
-GURL GetCatalogURL() {
-  const char kEspPath[] = "/esp.html";
-  std::string path(kEspPath);
+GURL GetCategoriesURL() {
+  const char kNtpJsonPath[] = "/getcategories";
+  std::string path(kNtpJsonPath);
 
   GURL base_url(GetBaseURL());
   GURL::Replacements replacements;

chrome/browser/android/explore_sites/url_util.h

@@ -12,12 +12,12 @@ namespace explore_sites {
 // Returns the base URL for the Explore Sites server.
 GURL GetBaseURL();
 
-// Returns the NTP JSON URL for the Explore Sites feature.
-GURL GetNtpURL();
-
-// Returns the ESP catalog URL for the Explore Sites feature.
+// Returns the URL for GetCatalog RPC.
 GURL GetCatalogURL();
 
+// Returns the URL for GetCategories RPC.
+GURL GetCategoriesURL();
+
 }  // namespace explore_sites
 
 #endif  // CHROME_BROWSER_ANDROID_EXPLORE_SITES_URL_UTIL_H_

chrome/browser/android/explore_sites/url_util_experimental.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/android/explore_sites/url_util_experimental.h"
+
+#include <string>
+
+#include "base/metrics/field_trial_params.h"
+#include "chrome/browser/android/chrome_feature_list.h"
+#include "url/gurl.h"
+
+namespace explore_sites {
+
+GURL GetBasePrototypeURL() {
+  const char kBaseURLOption[] = "experimental_base_url";
+  const char kDefaultBaseUrl[] =
+      "https://explore-sites-ux-research.appspot.com";
+  std::string field_trial_param = base::GetFieldTrialParamValueByFeature(
+      chrome::android::kExploreSites, kBaseURLOption);
+  if (field_trial_param.empty())
+    return GURL(kDefaultBaseUrl);
+  return GURL(field_trial_param);
+}
+
+GURL GetNtpPrototypeURL() {
+  const char kNtpJsonPath[] = "/ntp.json";
+  std::string path(kNtpJsonPath);
+
+  GURL base_url(GetBasePrototypeURL());
+  GURL::Replacements replacements;
+  replacements.SetPathStr(path);
+  return base_url.ReplaceComponents(replacements);
+}
+
+GURL GetCatalogPrototypeURL() {
+  const char kEspPath[] = "/esp.html";
+  std::string path(kEspPath);
+
+  GURL base_url(GetBasePrototypeURL());
+  GURL::Replacements replacements;
+  replacements.SetPathStr(path);
+  return base_url.ReplaceComponents(replacements);
+}
+
+}  // namespace explore_sites

chrome/browser/android/explore_sites/url_util_experimental.h

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_ANDROID_EXPLORE_SITES_URL_UTIL_EXPERIMENTAL_H_
+#define CHROME_BROWSER_ANDROID_EXPLORE_SITES_URL_UTIL_EXPERIMENTAL_H_
+
+#include "url/gurl.h"
+
+namespace explore_sites {
+
+// Returns the base URL for the Explore Sites server.
+GURL GetBasePrototypeURL();
+
+// Returns the NTP JSON URL for the Explore Sites feature.
+GURL GetNtpPrototypeURL();
+
+// Returns the ESP catalog URL for the Explore Sites feature.
+GURL GetCatalogPrototypeURL();
+
+}  // namespace explore_sites
+
+#endif  // CHROME_BROWSER_ANDROID_EXPLORE_SITES_URL_UTIL_EXPERIMENTAL_H_

chrome/test/BUILD.gn

@@ -2329,6 +2329,7 @@ test("unit_tests") {
     "../browser/android/download/available_offline_content_provider_unittest.cc",
     "../browser/android/download/download_manager_service_unittest.cc",
     "../browser/android/explore_sites/explore_sites_feature_unittest.cc",
+    "../browser/android/explore_sites/explore_sites_fetcher_unittest.cc",
     "../browser/android/explore_sites/explore_sites_schema_unittest.cc",
     "../browser/android/explore_sites/explore_sites_store_unittest.cc",
     "../browser/android/explore_sites/get_catalog_task_unittest.cc",