[AF]Class to download and parse JSON files from gstatic

Bug: 992491
Change-Id: Icf791568a56e5dafb9f280021aad4a0742520fe1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1733726
Commit-Queue: Anne Lim <annelim@google.com>
Reviewed-by: Martin Šrámek <msramek@chromium.org>
Reviewed-by: Marc Treib <treib@chromium.org>
Reviewed-by: Jared Saul <jsaul@google.com>
Cr-Commit-Position: refs/heads/master@{#688600}

chrome/browser/BUILD.gn

@@ -123,6 +123,8 @@ jumbo_split_static_library("browser") {
     "autofill/address_normalizer_factory.h",
     "autofill/autocomplete_history_manager_factory.cc",
     "autofill/autocomplete_history_manager_factory.h",
+    "autofill/autofill_gstatic_reader.cc",
+    "autofill/autofill_gstatic_reader.h",
     "autofill/autofill_profile_validator_factory.cc",
     "autofill/autofill_profile_validator_factory.h",
     "autofill/personal_data_manager_factory.cc",

chrome/browser/autofill/autofill_gstatic_reader.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/autofill/autofill_gstatic_reader.h"
+
+#include <list>
+#include <memory>
+#include <utility>
+
+#include "base/json/json_reader.h"
+#include "chrome/browser/browser_process.h"
+#include "chrome/browser/net/system_network_context_manager.h"
+#include "content/public/browser/browser_context.h"
+#include "content/public/browser/storage_partition.h"
+#include "net/base/load_flags.h"
+#include "net/traffic_annotation/network_traffic_annotation.h"
+#include "services/network/public/cpp/resource_request.h"
+#include "services/network/public/cpp/shared_url_loader_factory.h"
+
+namespace autofill {
+
+namespace {
+static const char kTokenizationBinRangeWhitelistKey[] =
+    "cpan_eligible_bin_wl_regex";
+static const char kTokenizationMerchantWhitelistKey[] =
+    "cpan_eligible_merchant_wl";
+static const char kTokenizationBinRangeWhitelistURL[] =
+    "https://www.gstatic.com/autofill/hourly/bins.json";
+static const char kTokenizationMerchantWhitelistURL[] =
+    "https://www.gstatic.com/autofill/weekly/merchants.json";
+static const size_t kMaxDownloadSize = 30 * 1024;
+}  // namespace
+
+AutofillGstaticReader::AutofillGstaticReader() {}
+
+AutofillGstaticReader::~AutofillGstaticReader() {}
+
+void AutofillGstaticReader::SetUp() {
+  if (!setup_called_) {
+    setup_called_ = true;
+    LoadDataAsList(GURL(kTokenizationBinRangeWhitelistURL),
+                   kTokenizationBinRangeWhitelistKey);
+    LoadDataAsList(GURL(kTokenizationMerchantWhitelistURL),
+                   kTokenizationMerchantWhitelistKey);
+  }
+}
+
+AutofillGstaticReader* AutofillGstaticReader::GetInstance() {
+  static base::NoDestructor<AutofillGstaticReader> instance;
+  return instance.get();
+}
+
+std::vector<std::string>
+AutofillGstaticReader::GetTokenizationMerchantWhitelist() const {
+  return tokenization_merchant_whitelist_;
+}
+
+std::vector<std::string>
+AutofillGstaticReader::GetTokenizationBinRangesWhitelist() const {
+  return tokenization_bin_range_whitelist_;
+}
+
+void AutofillGstaticReader::LoadDataAsList(const GURL& url,
+                                           const std::string& key) {
+  DCHECK(setup_called_);
+  auto resource_request = std::make_unique<network::ResourceRequest>();
+  resource_request->credentials_mode = network::mojom::CredentialsMode::kOmit;
+  resource_request->url = url;
+  resource_request->method = "GET";
+  net::NetworkTrafficAnnotationTag traffic_annotation =
+      net::DefineNetworkTrafficAnnotation("load_autofill_gstatic_data", R"(
+        semantics {
+          sender: "Autofill"
+          description:
+            "Downloads data used to decide when to offer Autofill features, "
+            "such as whitelists of eligible websites."
+          trigger: "Triggered once on Chrome startup."
+          data: "None"
+          destination: GOOGLE_OWNED_SERVICE
+        }
+        policy {
+          cookies_allowed: NO
+          setting:
+            "You can disable this feature by disabling the Payments and "
+            "Addresses settings in Chrome's settings under Autofill. This "
+            "feature is always enabled unless both Payments AND Addresses "
+            "Autofill are disabled."
+          chrome_policy {
+            AutofillCreditCardEnabled {
+                policy_options {mode: MANDATORY}
+                AutofillCreditCardEnabled: true
+            }
+          }
+          chrome_policy {
+            AutofillAddressEnabled {
+                policy_options {mode: MANDATORY}
+                AutofillAddressEnabled: true
+            }
+          }
+        }
+        comments: "Both the AutofillAddressEnabled and "
+          "AutofillCreditCardEnabled policies needs to be disabled for this "
+          "network request to be disabled.")");
+  auto simple_loader = network::SimpleURLLoader::Create(
+      std::move(resource_request), traffic_annotation);
+  simple_loader->SetAllowHttpErrorResults(true);
+
+  // Transfer ownership of the loader into |url_loaders_|. Temporarily hang
+  // onto the raw pointer to kick off the request;
+  // transferring ownership (std::move) invalidates the |simple_loader|
+  // variable.
+  auto* raw_simple_loader = simple_loader.get();
+  url_loaders_.push_back(std::move(simple_loader));
+  if (g_browser_process->system_network_context_manager()->HasInstance()) {
+    raw_simple_loader->DownloadToString(
+        g_browser_process->system_network_context_manager()
+            ->GetSharedURLLoaderFactory()
+            .get(),
+        base::BindOnce(&AutofillGstaticReader::OnSimpleLoaderComplete,
+                       base::Unretained(this), --url_loaders_.end(), key),
+        kMaxDownloadSize);
+  }
+}
+
+void AutofillGstaticReader::OnSimpleLoaderComplete(
+    std::list<std::unique_ptr<network::SimpleURLLoader>>::iterator it,
+    const std::string& key,
+    std::unique_ptr<std::string> response_body) {
+  // Move the loader out of the active loaders list.
+  std::unique_ptr<network::SimpleURLLoader> simple_loader = std::move(*it);
+  url_loaders_.erase(it);
+  SetListClassVariable(ParseListJSON(std::move(response_body), key), key);
+}
+
+// static
+std::vector<std::string> AutofillGstaticReader::ParseListJSON(
+    std::unique_ptr<std::string> response_body,
+    const std::string& key) {
+  if (!response_body)
+    return {};
+  base::Optional<base::Value> data = base::JSONReader::Read(*response_body);
+  if (data == base::nullopt || !data->is_dict())
+    return {};
+  base::Value* raw_result = data->FindKey(key);
+  if (!raw_result || !raw_result->is_list())
+    return {};
+  std::vector<std::string> result;
+  for (const base::Value& value : raw_result->GetList()) {
+    if (value.is_string())
+      result.push_back(value.GetString());
+  }
+  return result;
+}
+
+void AutofillGstaticReader::SetListClassVariable(
+    std::vector<std::string> result,
+    const std::string& key) {
+  if (key == kTokenizationBinRangeWhitelistKey) {
+    tokenization_bin_range_whitelist_ = result;
+  } else if (key == kTokenizationMerchantWhitelistKey) {
+    tokenization_merchant_whitelist_ = result;
+  }
+}
+
+}  // namespace autofill

chrome/browser/autofill/autofill_gstatic_reader.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROME_BROWSER_AUTOFILL_AUTOFILL_GSTATIC_READER_H_
+#define CHROME_BROWSER_AUTOFILL_AUTOFILL_GSTATIC_READER_H_
+
+#include <list>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "chrome/browser/profiles/profile.h"
+#include "components/keyed_service/core/keyed_service.h"
+#include "services/network/public/cpp/simple_url_loader.h"
+
+namespace autofill {
+
+// Fetches and parses JSON data from Gstatic URLs, and stores them
+// so that they can be accessed later.
+class AutofillGstaticReader {
+ public:
+  AutofillGstaticReader();
+
+  ~AutofillGstaticReader();
+
+  // Sets up the class by loading data from the Gstatic URLs if setup hasn't
+  // been attempted yet. Should be called only when ChromeAutofillClient
+  // starts up and Autofill is enabled.
+  // After it has been called, subsequent calls won't have any effect.
+  void SetUp();
+
+  static AutofillGstaticReader* GetInstance();
+
+  // Returns list of merchants whitelisted for cloud tokenization. An empty list
+  // will be returned if Setup() failed, hasn't been called yet, or hasn't
+  // finished downloading the whitelist.
+  std::vector<std::string> GetTokenizationMerchantWhitelist() const;
+
+  // Returns list of BIN ranges of cards whitelisted for cloud tokenization. An
+  // empty list will be returned if Setup() failed, hasn't been called yet, or
+  // hasn't finished downloading the whitelist.
+  std::vector<std::string> GetTokenizationBinRangesWhitelist() const;
+
+ private:
+  FRIEND_TEST_ALL_PREFIXES(AutofillGstaticReaderTest,
+                           ParseListJSON_InvalidKeyNotParsed);
+  FRIEND_TEST_ALL_PREFIXES(AutofillGstaticReaderTest,
+                           ParseListJSON_NonDictionaryNotParsed);
+  FRIEND_TEST_ALL_PREFIXES(AutofillGstaticReaderTest,
+                           ParseListJSON_NonStringListEntryNotParsed);
+  FRIEND_TEST_ALL_PREFIXES(AutofillGstaticReaderTest,
+                           ParseListJSON_ValidResponseGetsParsed);
+
+  // Fetches data stored at |url| which have the following JSON format:
+  // { "key": ["list_item_one", "list_item2", ...]}
+  // The entry with |key| as key will be saved if it is present.
+  void LoadDataAsList(const GURL& url, const std::string& key);
+
+  // Callback which receives the content of |url| from LoadDataAsList(~).
+  void OnSimpleLoaderComplete(
+      std::list<std::unique_ptr<network::SimpleURLLoader>>::iterator it,
+      const std::string& key,
+      std::unique_ptr<std::string> response_body);
+
+  // Parses and returns list of strings which are in the format of a
+  // list value to a JSON key.
+  static std::vector<std::string> ParseListJSON(
+      std::unique_ptr<std::string> response_body,
+      const std::string& key);
+
+  void SetListClassVariable(std::vector<std::string> result,
+                            const std::string& key);
+
+  bool setup_called_ = false;
+
+  // BIN ranges which are eligible for cloud tokenization.
+  std::vector<std::string> tokenization_bin_range_whitelist_;
+
+  // Merchant domains which are eligible for cloud tokenization.
+  std::vector<std::string> tokenization_merchant_whitelist_;
+
+  // Loaders used for the processing the requests. Each loader is removed upon
+  // completion.
+  std::list<std::unique_ptr<network::SimpleURLLoader>> url_loaders_;
+};
+}  // namespace autofill
+
+#endif  // CHROME_BROWSER_AUTOFILL_AUTOFILL_GSTATIC_READER_H_

chrome/browser/autofill/autofill_gstatic_reader_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/autofill/autofill_gstatic_reader.h"
+
+#include <utility>
+
+#include "base/test/scoped_task_environment.h"
+#include "chrome/test/base/testing_profile.h"
+#include "content/public/test/test_browser_thread_bundle.h"
+#include "services/network/test/test_url_loader_factory.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace autofill {
+
+class AutofillGstaticReaderTest : public ::testing::Test {
+ public:
+  AutofillGstaticReaderTest() {}
+
+ private:
+  content::TestBrowserThreadBundle thread_bundle_;
+  network::TestURLLoaderFactory test_url_loader_factory_;
+};
+
+TEST_F(AutofillGstaticReaderTest, ParseListJSON_ValidResponseGetsParsed) {
+  std::unique_ptr<std::string> response = std::make_unique<std::string>(
+      "{\"list_data_key\": [\"list_item_one\", "
+      "\"list_item_two\"]}");
+  std::vector<std::string> merchant_whitelist =
+      AutofillGstaticReader::GetInstance()->ParseListJSON(std::move(response),
+                                                          "list_data_key");
+  EXPECT_EQ(merchant_whitelist.size(), 2U);
+  EXPECT_EQ(merchant_whitelist[0], "list_item_one");
+  EXPECT_EQ(merchant_whitelist[1], "list_item_two");
+}
+
+TEST_F(AutofillGstaticReaderTest, ParseListJSON_InvalidKeyNotParsed) {
+  std::unique_ptr<std::string> response = std::make_unique<std::string>(
+      "{\"randomKey\": [\"list_item_one\", \"list_item_two\"]}");
+  std::vector<std::string> merchant_whitelist =
+      AutofillGstaticReader::GetInstance()->ParseListJSON(std::move(response),
+                                                          "list_data_key");
+  // "list_data_key" isn't a key in |response|, so we expect to return an empty
+  // list.
+  EXPECT_EQ(merchant_whitelist.size(), 0U);
+}
+
+TEST_F(AutofillGstaticReaderTest, ParseListJSON_NonStringListEntryNotParsed) {
+  std::unique_ptr<std::string> response = std::make_unique<std::string>(
+      "{\"list_data_key\": [1, \"list_item_two\"]}");
+  std::vector<std::string> merchant_whitelist =
+      AutofillGstaticReader::GetInstance()->ParseListJSON(std::move(response),
+                                                          "list_data_key");
+  EXPECT_EQ(merchant_whitelist.size(), 1U);
+  EXPECT_EQ(merchant_whitelist[0], "list_item_two");
+}
+
+TEST_F(AutofillGstaticReaderTest, ParseListJSON_NonDictionaryNotParsed) {
+  std::unique_ptr<std::string> response =
+      std::make_unique<std::string>("list_item_one");
+  std::vector<std::string> merchant_whitelist =
+      AutofillGstaticReader::GetInstance()->ParseListJSON(std::move(response),
+                                                          "list_data_key");
+  EXPECT_EQ(merchant_whitelist.size(), 0U);
+}
+
+}  // namespace autofill

chrome/browser/ui/autofill/chrome_autofill_client.cc

@@ -13,6 +13,7 @@
 #include "base/strings/utf_string_conversions.h"
 #include "chrome/browser/autofill/address_normalizer_factory.h"
 #include "chrome/browser/autofill/autocomplete_history_manager_factory.h"
+#include "chrome/browser/autofill/autofill_gstatic_reader.h"
 #include "chrome/browser/autofill/personal_data_manager_factory.h"
 #include "chrome/browser/autofill/risk_util.h"
 #include "chrome/browser/autofill/strike_database_factory.h"
@@ -566,6 +567,8 @@ ChromeAutofillClient::ChromeAutofillClient(content::WebContents* web_contents)
           user_prefs::UserPrefs::Get(web_contents->GetBrowserContext()),
           Profile::FromBrowserContext(web_contents->GetBrowserContext())
               ->IsOffTheRecord()) {
+  if (::autofill::prefs::IsAutofillEnabled(GetPrefs()))
+    AutofillGstaticReader::GetInstance()->SetUp();
   // TODO(crbug.com/928595): Replace the closure with a callback to the renderer
   // that indicates if log messages should be sent from the renderer.
   log_manager_ =

chrome/test/BUILD.gn

@@ -2803,6 +2803,7 @@ test("unit_tests") {
     "../browser/autocomplete/chrome_autocomplete_scheme_classifier_unittest.cc",
     "../browser/autocomplete/search_provider_unittest.cc",
     "../browser/autocomplete/shortcuts_provider_extension_unittest.cc",
+    "../browser/autofill/autofill_gstatic_reader_unittest.cc",
     "../browser/availability/availability_prober_unittest.cc",
     "../browser/background_fetch/background_fetch_delegate_impl_unittest.cc",
     "../browser/background_fetch/background_fetch_permission_context_unittest.cc",

tools/traffic_annotation/summary/annotations.xml

@@ -143,6 +143,7 @@ Refer to README.md for content description and update process.
  <item id="invalidation_service" hash_code="72354423" type="0" content_hash_code="78425687" os_list="linux,windows" file_path="components/invalidation/impl/gcm_network_channel.cc"/>
  <item id="kids_chrome_management_client_classify_url" hash_code="109987793" type="0" deprecated="2019-07-30" content_hash_code="112740597" file_path=""/>
  <item id="lib_address_input" hash_code="50816767" type="0" content_hash_code="57977576" os_list="linux,windows" file_path="third_party/libaddressinput/chromium/chrome_metadata_source.cc"/>
+ <item id="load_autofill_gstatic_data" hash_code="119416099" type="0" content_hash_code="8433621" os_list="linux,windows" file_path="chrome/browser/autofill/autofill_gstatic_reader.cc"/>
  <item id="logo_service" hash_code="35473769" type="0" content_hash_code="20271299" os_list="linux,windows" file_path="components/search_provider_logos/logo_service_impl.cc"/>
  <item id="logo_tracker" hash_code="36859107" type="0" deprecated="2018-12-07" content_hash_code="67588075" file_path=""/>
  <item id="lookup_single_password_leak" hash_code="16927377" type="0" content_hash_code="12158296" os_list="linux,windows" file_path="components/password_manager/core/browser/leak_detection/leak_detection_request.cc"/>