Add ContextualJsonRequest

ContextualJsonRequest can build and send a Json request to fetch
contextual suggestions.
Contextual suggestions are suggestions based on a given URL.
A unit test verifies that a valid Json request is built.

Bug: n/a
Change-Id: Ic19def03c4080b42117624e9452376abbb68ffa5
Reviewed-on: https://chromium-review.googlesource.com/577540
Commit-Queue: Andre Gaschler <gaschler@chromium.org>
Reviewed-by: Markus Heintz <markusheintz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#488223}

components/ntp_snippets/BUILD.gn

@@ -62,6 +62,8 @@ static_library("ntp_snippets") {
     "reading_list/reading_list_suggestions_provider.h",
     "remote/cached_image_fetcher.cc",
     "remote/cached_image_fetcher.h",
+    "remote/contextual_json_request.cc",
+    "remote/contextual_json_request.h",
     "remote/json_request.cc",
     "remote/json_request.h",
     "remote/json_to_categories.cc",
@@ -174,6 +176,7 @@ source_set("unit_tests") {
     "physical_web_pages/physical_web_page_suggestions_provider_unittest.cc",
     "reading_list/reading_list_suggestions_provider_unittest.cc",
     "remote/cached_image_fetcher_unittest.cc",
+    "remote/contextual_json_request_unittest.cc",
     "remote/json_request_unittest.cc",
     "remote/prefetched_pages_tracker_impl_unittest.cc",
     "remote/remote_suggestion_unittest.cc",

components/ntp_snippets/remote/contextual_json_request.cc

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/ntp_snippets/remote/contextual_json_request.h"
+
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include "base/json/json_writer.h"
+#include "base/memory/ptr_util.h"
+#include "base/strings/stringprintf.h"
+#include "base/values.h"
+#include "components/data_use_measurement/core/data_use_user_data.h"
+#include "components/ntp_snippets/category_info.h"
+#include "components/ntp_snippets/features.h"
+#include "components/strings/grit/components_strings.h"
+#include "components/variations/net/variations_http_headers.h"
+#include "components/variations/variations_associated_data.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_response_headers.h"
+#include "net/http/http_status_code.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_request_context_getter.h"
+#include "third_party/icu/source/common/unicode/uloc.h"
+#include "third_party/icu/source/common/unicode/utypes.h"
+
+using net::URLFetcher;
+using net::URLRequestContextGetter;
+using net::HttpRequestHeaders;
+using net::URLRequestStatus;
+
+namespace ntp_snippets {
+
+namespace internal {
+
+namespace {
+
+const int k5xxRetries = 2;
+
+}  // namespace
+
+ContextualJsonRequest::ContextualJsonRequest(const ParseJSONCallback& callback)
+    : parse_json_callback_(callback), weak_ptr_factory_(this) {}
+
+ContextualJsonRequest::~ContextualJsonRequest() {
+  DLOG_IF(ERROR, !request_completed_callback_.is_null())
+      << "The CompletionCallback was never called!";
+}
+
+void ContextualJsonRequest::Start(CompletedCallback callback) {
+  request_completed_callback_ = std::move(callback);
+  url_fetcher_->Start();
+}
+
+std::string ContextualJsonRequest::GetResponseString() const {
+  std::string response;
+  url_fetcher_->GetResponseAsString(&response);
+  return response;
+}
+
+// URLFetcherDelegate overrides
+void ContextualJsonRequest::OnURLFetchComplete(const net::URLFetcher* source) {
+  DCHECK_EQ(url_fetcher_.get(), source);
+  const URLRequestStatus& status = url_fetcher_->GetStatus();
+  int response = url_fetcher_->GetResponseCode();
+  // TODO(gaschler): Add UMA metrics for response status code
+
+  if (!status.is_success()) {
+    std::move(request_completed_callback_)
+        .Run(/*result=*/nullptr, FetchResult::URL_REQUEST_STATUS_ERROR,
+             /*error_details=*/base::StringPrintf(" %d", status.error()));
+  } else if (response != net::HTTP_OK) {
+    // TODO(jkrcal): https://crbug.com/609084
+    // We need to deal with the edge case again where the auth
+    // token expires just before we send the request (in which case we need to
+    // fetch a new auth token). We should extract that into a common class
+    // instead of adding it to every single class that uses auth tokens.
+    std::move(request_completed_callback_)
+        .Run(/*result=*/nullptr, FetchResult::HTTP_ERROR,
+             /*error_details=*/base::StringPrintf(" %d", response));
+  } else {
+    ParseJsonResponse();
+  }
+}
+
+void ContextualJsonRequest::ParseJsonResponse() {
+  std::string json_string;
+  bool stores_result_to_string =
+      url_fetcher_->GetResponseAsString(&json_string);
+  DCHECK(stores_result_to_string);
+
+  parse_json_callback_.Run(json_string,
+                           base::Bind(&ContextualJsonRequest::OnJsonParsed,
+                                      weak_ptr_factory_.GetWeakPtr()),
+                           base::Bind(&ContextualJsonRequest::OnJsonError,
+                                      weak_ptr_factory_.GetWeakPtr()));
+}
+
+void ContextualJsonRequest::OnJsonParsed(std::unique_ptr<base::Value> result) {
+  std::move(request_completed_callback_)
+      .Run(std::move(result), FetchResult::SUCCESS,
+           /*error_details=*/std::string());
+}
+
+void ContextualJsonRequest::OnJsonError(const std::string& error) {
+  std::string json_string;
+  url_fetcher_->GetResponseAsString(&json_string);
+  LOG(WARNING) << "Received invalid JSON (" << error << "): " << json_string;
+  std::move(request_completed_callback_)
+      .Run(/*result=*/nullptr, FetchResult::JSON_PARSE_ERROR,
+           /*error_details=*/base::StringPrintf(" (error %s)", error.c_str()));
+}
+
+ContextualJsonRequest::Builder::Builder() = default;
+ContextualJsonRequest::Builder::Builder(ContextualJsonRequest::Builder&&) =
+    default;
+ContextualJsonRequest::Builder::~Builder() = default;
+
+std::unique_ptr<ContextualJsonRequest> ContextualJsonRequest::Builder::Build()
+    const {
+  DCHECK(url_request_context_getter_);
+  auto request = base::MakeUnique<ContextualJsonRequest>(parse_json_callback_);
+  std::string body = BuildBody();
+  std::string headers = BuildHeaders();
+  request->url_fetcher_ = BuildURLFetcher(request.get(), headers, body);
+
+  // Log the request for debugging network issues.
+  VLOG(1) << "Sending a NTP snippets request to " << url_ << ":\n"
+          << headers << "\n"
+          << body;
+
+  return request;
+}
+
+ContextualJsonRequest::Builder&
+ContextualJsonRequest::Builder::SetAuthentication(
+    const std::string& account_id,
+    const std::string& auth_header) {
+  auth_header_ = auth_header;
+  return *this;
+}
+
+ContextualJsonRequest::Builder&
+ContextualJsonRequest::Builder::SetParseJsonCallback(
+    ParseJSONCallback callback) {
+  parse_json_callback_ = callback;
+  return *this;
+}
+
+ContextualJsonRequest::Builder& ContextualJsonRequest::Builder::SetUrl(
+    const GURL& url) {
+  url_ = url;
+  return *this;
+}
+
+ContextualJsonRequest::Builder&
+ContextualJsonRequest::Builder::SetUrlRequestContextGetter(
+    const scoped_refptr<net::URLRequestContextGetter>& context_getter) {
+  url_request_context_getter_ = context_getter;
+  return *this;
+}
+
+ContextualJsonRequest::Builder& ContextualJsonRequest::Builder::SetContentUrl(
+    const GURL& url) {
+  content_url_ = url;
+  return *this;
+}
+
+std::string ContextualJsonRequest::Builder::BuildHeaders() const {
+  net::HttpRequestHeaders headers;
+  headers.SetHeader("Content-Type", "application/json; charset=UTF-8");
+  if (!auth_header_.empty()) {
+    headers.SetHeader("Authorization", auth_header_);
+  }
+  // Add X-Client-Data header with experiment IDs from field trials.
+  // Note: It's OK to pass |is_signed_in| false if it's unknown, as it does
+  // not affect transmission of experiments coming from the variations server.
+  bool is_signed_in = false;
+  variations::AppendVariationHeaders(url_,
+                                     false,  // incognito
+                                     false,  // uma_enabled
+                                     is_signed_in, &headers);
+  return headers.ToString();
+}
+
+std::string ContextualJsonRequest::Builder::BuildBody() const {
+  auto request = base::MakeUnique<base::DictionaryValue>();
+
+  request->SetString("url", content_url_.spec());
+  auto categories = base::MakeUnique<base::ListValue>();
+  categories->AppendString("RELATED_ARTICLES");
+  categories->AppendString("PUBLIC_DEBATE");
+  request->Set("categories", std::move(categories));
+
+  std::string request_json;
+  bool success = base::JSONWriter::WriteWithOptions(
+      *request, base::JSONWriter::OPTIONS_PRETTY_PRINT, &request_json);
+  DCHECK(success);
+  return request_json;
+}
+
+std::unique_ptr<net::URLFetcher>
+ContextualJsonRequest::Builder::BuildURLFetcher(
+    net::URLFetcherDelegate* delegate,
+    const std::string& headers,
+    const std::string& body) const {
+  net::NetworkTrafficAnnotationTag traffic_annotation =
+      net::DefineNetworkTrafficAnnotation("ntp_snippets_fetch", R"(
+        semantics {
+          sender: "New Tab Page Contextual Suggestions Fetch"
+          description:
+            "Chromium can show contextual suggestions that are related to the "
+            "currently visited page on the New Tab page. "
+          trigger:
+            "Triggered when Home sheet is pulled up."
+          data:
+            "The Chromium UI language, as well as a second language the user "
+            "understands, based on translate::LanguageModel. For signed-in "
+            "users, the requests is authenticated."
+          destination: GOOGLE_OWNED_SERVICE
+        }
+        policy {
+          cookies_allowed: false
+          setting:
+            "This feature can be disabled by the flag "
+            "contextual-suggestions-carousel."
+        })");
+  std::unique_ptr<net::URLFetcher> url_fetcher = net::URLFetcher::Create(
+      url_, net::URLFetcher::POST, delegate, traffic_annotation);
+  url_fetcher->SetRequestContext(url_request_context_getter_.get());
+  url_fetcher->SetLoadFlags(net::LOAD_DO_NOT_SEND_COOKIES |
+                            net::LOAD_DO_NOT_SAVE_COOKIES);
+  data_use_measurement::DataUseUserData::AttachToFetcher(
+      url_fetcher.get(),
+      data_use_measurement::DataUseUserData::NTP_SNIPPETS_SUGGESTIONS);
+
+  url_fetcher->SetExtraRequestHeaders(headers);
+  url_fetcher->SetUploadData("application/json", body);
+
+  // Fetchers are sometimes cancelled because a network change was detected.
+  url_fetcher->SetAutomaticallyRetryOnNetworkChanges(3);
+  url_fetcher->SetMaxRetriesOn5xx(k5xxRetries);
+  return url_fetcher;
+}
+
+}  // namespace internal
+
+}  // namespace ntp_snippets

components/ntp_snippets/remote/contextual_json_request.h

+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef COMPONENTS_NTP_SNIPPETS_REMOTE_CONTEXTUAL_JSON_REQUEST_H_
+#define COMPONENTS_NTP_SNIPPETS_REMOTE_CONTEXTUAL_JSON_REQUEST_H_
+
+#include <memory>
+#include <string>
+#include <utility>
+
+#include "base/callback.h"
+#include "base/memory/weak_ptr.h"
+#include "components/ntp_snippets/remote/json_request.h"
+#include "components/ntp_snippets/status.h"
+#include "google_apis/gaia/oauth2_token_service.h"
+#include "net/http/http_request_headers.h"
+
+namespace base {
+class Value;
+}  // namespace base
+
+namespace ntp_snippets {
+
+namespace internal {
+
+// A request to query contextual suggestions.
+class ContextualJsonRequest : public net::URLFetcherDelegate {
+ public:
+  // A client can expect error_details only, if there was any error during the
+  // fetching or parsing. In successful cases, it will be an empty string.
+  using CompletedCallback =
+      base::OnceCallback<void(std::unique_ptr<base::Value> result,
+                              FetchResult result_code,
+                              const std::string& error_details)>;
+
+  // Builds authenticated and non-authenticated ContextualJsonRequests.
+  class Builder {
+   public:
+    Builder();
+    Builder(Builder&&);
+    ~Builder();
+
+    // Builds a Request object that contains all data to fetch new snippets.
+    std::unique_ptr<ContextualJsonRequest> Build() const;
+
+    Builder& SetAuthentication(const std::string& account_id,
+                               const std::string& auth_header);
+    Builder& SetParseJsonCallback(ParseJSONCallback callback);
+    Builder& SetUrl(const GURL& url);
+    Builder& SetUrlRequestContextGetter(
+        const scoped_refptr<net::URLRequestContextGetter>& context_getter);
+    Builder& SetContentUrl(const GURL& url);
+
+    // These preview methods allow to inspect the Request without exposing it
+    // publicly.
+    std::string PreviewRequestBodyForTesting() { return BuildBody(); }
+    std::string PreviewRequestHeadersForTesting() { return BuildHeaders(); }
+
+   private:
+    std::string BuildHeaders() const;
+    std::string BuildBody() const;
+    std::unique_ptr<net::URLFetcher> BuildURLFetcher(
+        net::URLFetcherDelegate* request,
+        const std::string& headers,
+        const std::string& body) const;
+
+    std::string auth_header_;
+    ParseJSONCallback parse_json_callback_;
+    GURL url_;
+    scoped_refptr<net::URLRequestContextGetter> url_request_context_getter_;
+
+    // The URL for which to fetch contextual suggestions for.
+    GURL content_url_;
+
+    DISALLOW_COPY_AND_ASSIGN(Builder);
+  };
+
+  ContextualJsonRequest(const ParseJSONCallback& callback);
+  ContextualJsonRequest(ContextualJsonRequest&&);
+  ~ContextualJsonRequest() override;
+
+  void Start(CompletedCallback callback);
+
+  std::string GetResponseString() const;
+
+ private:
+  // URLFetcherDelegate implementation.
+  void OnURLFetchComplete(const net::URLFetcher* source) override;
+
+  void ParseJsonResponse();
+  void OnJsonParsed(std::unique_ptr<base::Value> result);
+  void OnJsonError(const std::string& error);
+
+  // The fetcher for downloading the snippets. Only non-null if a fetch is
+  // currently ongoing.
+  std::unique_ptr<net::URLFetcher> url_fetcher_;
+
+  // This callback is called to parse a json string. It contains callbacks for
+  // error and success cases.
+  ParseJSONCallback parse_json_callback_;
+
+  // The callback to notify when URLFetcher finished and results are available.
+  CompletedCallback request_completed_callback_;
+
+  base::WeakPtrFactory<ContextualJsonRequest> weak_ptr_factory_;
+
+  DISALLOW_COPY_AND_ASSIGN(ContextualJsonRequest);
+};
+
+}  // namespace internal
+
+}  // namespace ntp_snippets
+
+#endif  // COMPONENTS_NTP_SNIPPETS_REMOTE_CONTEXTUAL_JSON_REQUEST_H_

components/ntp_snippets/remote/contextual_json_request_unittest.cc

+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "components/ntp_snippets/remote/contextual_json_request.h"
+
+#include <utility>
+
+#include "base/json/json_reader.h"
+#include "base/memory/ptr_util.h"
+#include "base/message_loop/message_loop.h"
+#include "base/test/test_mock_time_task_runner.h"
+#include "base/values.h"
+#include "components/ntp_snippets/features.h"
+#include "net/url_request/url_request_test_util.h"
+#include "testing/gmock/include/gmock/gmock.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace ntp_snippets {
+
+namespace internal {
+
+namespace {
+
+using testing::_;
+using testing::Eq;
+using testing::StrEq;
+
+MATCHER_P(EqualsJSON, json, "equals JSON") {
+  std::unique_ptr<base::Value> expected = base::JSONReader::Read(json);
+  if (!expected) {
+    *result_listener << "INTERNAL ERROR: couldn't parse expected JSON";
+    return false;
+  }
+
+  std::string err_msg;
+  int err_line, err_col;
+  std::unique_ptr<base::Value> actual = base::JSONReader::ReadAndReturnError(
+      arg, base::JSON_PARSE_RFC, nullptr, &err_msg, &err_line, &err_col);
+  if (!actual) {
+    *result_listener << "input:" << err_line << ":" << err_col << ": "
+                     << "parse error: " << err_msg;
+    return false;
+  }
+  return base::Value::Equals(actual.get(), expected.get());
+}
+
+}  // namespace
+
+class ContextualJsonRequestTest : public testing::Test {
+ public:
+  ContextualJsonRequestTest()
+      : request_context_getter_(
+            new net::TestURLRequestContextGetter(loop_.task_runner())) {}
+
+  ContextualJsonRequest::Builder CreateDefaultBuilder() {
+    ContextualJsonRequest::Builder builder;
+    builder.SetUrl(GURL("http://valid-url.test"))
+        .SetUrlRequestContextGetter(request_context_getter_.get());
+    return builder;
+  }
+
+ private:
+  base::MessageLoop loop_;
+  scoped_refptr<net::TestURLRequestContextGetter> request_context_getter_;
+
+  DISALLOW_COPY_AND_ASSIGN(ContextualJsonRequestTest);
+};
+
+TEST_F(ContextualJsonRequestTest, AuthenticatedRequest) {
+  ContextualJsonRequest::Builder builder = CreateDefaultBuilder();
+  builder.SetAuthentication("0BFUSGAIA", "headerstuff")
+      .SetContentUrl(GURL("http://my-url.test"))
+      .Build();
+
+  EXPECT_THAT(builder.PreviewRequestHeadersForTesting(),
+              StrEq("Content-Type: application/json; charset=UTF-8\r\n"
+                    "Authorization: headerstuff\r\n"
+                    "\r\n"));
+  EXPECT_THAT(builder.PreviewRequestBodyForTesting(),
+              EqualsJSON("{"
+                         "   \"categories\": [ \"RELATED_ARTICLES\","
+                         " \"PUBLIC_DEBATE\" ],"
+                         "   \"url\": \"http://my-url.test/\""
+                         "}"));
+}
+
+}  // namespace internal
+
+}  // namespace ntp_snippets