Move ExpandLanguageList from chrome/ to net/

Such that we can use it from android_webview/.

Bug: 882587, 737232
Change-Id: I0006cabbdbb31dfbf1b36a67dd226fe54afe76a8
Reviewed-on: https://chromium-review.googlesource.com/c/1274598Reviewed-by: Matt Menke <mmenke@chromium.org>
Reviewed-by: Justin DeWitt <dewittj@chromium.org>
Commit-Queue: Changwan Ryu <changwan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#600196}

chrome/browser/BUILD.gn

@@ -805,8 +805,6 @@ jumbo_split_static_library("browser") {
     "native_window_notification_source.h",
     "navigation_predictor/navigation_predictor.cc",
     "navigation_predictor/navigation_predictor.h",
-    "net/chrome_accept_language_settings.cc",
-    "net/chrome_accept_language_settings.h",
     "net/chrome_cookie_notification_details.h",
     "net/chrome_extensions_network_delegate.cc",
     "net/chrome_extensions_network_delegate.h",

chrome/browser/android/explore_sites/explore_sites_fetcher.cc

@@ -22,7 +22,6 @@
 #include "chrome/browser/android/explore_sites/explore_sites_types.h"
 #include "chrome/browser/android/explore_sites/url_util.h"
 #include "chrome/browser/browser_process.h"
-#include "chrome/browser/net/chrome_accept_language_settings.h"
 #include "chrome/common/channel_info.h"
 #include "components/variations/service/variations_service.h"
 #include "components/version_info/version_info.h"

chrome/browser/net/chrome_accept_language_settings.cc

-// Copyright (c) 2012 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "chrome/browser/net/chrome_accept_language_settings.h"
-
-#include <unordered_set>
-
-#include "base/feature_list.h"
-#include "base/strings/string_piece.h"
-#include "base/strings/string_split.h"
-#include "base/strings/string_util.h"
-#include "base/strings/stringprintf.h"
-#include "chrome/common/chrome_features.h"
-#include "net/http/http_util.h"
-
-namespace chrome_accept_language_settings {
-namespace {
-
-// Helper class that builds the list of languages for the Accept-Language
-// headers.
-// The output is a comma-separated list of languages as string.
-// Duplicates are removed.
-class AcceptLanguageBuilder {
- public:
-  // Adds a language to the string.
-  // Duplicates are ignored.
-  void AddLanguageCode(const std::string& language) {
-    if (seen_.find(language) == seen_.end()) {
-      if (str_.empty()) {
-        base::StringAppendF(&str_, "%s", language.c_str());
-      } else {
-        base::StringAppendF(&str_, ",%s", language.c_str());
-      }
-      seen_.insert(language);
-    }
-  }
-
-  // Returns the string constructed up to this point.
-  std::string GetString() const { return str_; }
-
- private:
-  // The string that contains the list of languages, comma-separated.
-  std::string str_;
-  // Set the remove duplicates.
-  std::unordered_set<std::string> seen_;
-};
-
-// Extract the base language code from a language code.
-// If there is no '-' in the code, the original code is returned.
-std::string GetBaseLanguageCode(const std::string& language_code) {
-  const std::vector<std::string> tokens = base::SplitString(
-      language_code, "-", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
-  return tokens.empty() ? "" : tokens[0];
-}
-
-}  // namespace
-
-std::string ComputeAcceptLanguageFromPref(const std::string& language_pref) {
-  std::string accept_languages_str =
-      base::FeatureList::IsEnabled(features::kUseNewAcceptLanguageHeader)
-          ? ExpandLanguageList(language_pref)
-          : language_pref;
-  return net::HttpUtil::GenerateAcceptLanguageHeader(accept_languages_str);
-}
-
-std::string ExpandLanguageList(const std::string& language_prefs) {
-  const std::vector<std::string> languages = base::SplitString(
-      language_prefs, ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
-
-  if (languages.empty())
-    return "";
-
-  AcceptLanguageBuilder builder;
-
-  const int size = languages.size();
-  for (int i = 0; i < size; ++i) {
-    const std::string& language = languages[i];
-    builder.AddLanguageCode(language);
-
-    // Extract the base language
-    const std::string& base_language = GetBaseLanguageCode(language);
-
-    // Look ahead and add the base language if the next language is not part
-    // of the same family.
-    const int j = i + 1;
-    if (j >= size || GetBaseLanguageCode(languages[j]) != base_language) {
-      builder.AddLanguageCode(base_language);
-    }
-  }
-
-  return builder.GetString();
-}
-
-}  // namespace chrome_accept_language_settings

chrome/browser/net/chrome_accept_language_settings.h

-// Copyright (c) 2012 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef CHROME_BROWSER_NET_CHROME_ACCEPT_LANGUAGE_SETTINGS_H_
-#define CHROME_BROWSER_NET_CHROME_ACCEPT_LANGUAGE_SETTINGS_H_
-
-#include <string>
-
-namespace chrome_accept_language_settings {
-
-// Given value of prefs::kAcceptLanguages pref, computes the corresponding
-// Accept-Language header to send.
-std::string ComputeAcceptLanguageFromPref(const std::string& language_pref);
-
-// Adds the base language if a corresponding language+region code is present.
-std::string ExpandLanguageList(const std::string& language_prefs);
-
-}  // namespace chrome_accept_language_settings
-
-#endif  // CHROME_BROWSER_NET_CHROME_ACCEPT_LANGUAGE_SETTINGS_H_

chrome/browser/net/chrome_accept_language_settings_unittest.cc

-// Copyright (c) 2017 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#include "chrome/browser/net/chrome_accept_language_settings.h"
-
-#include "testing/gtest/include/gtest/gtest.h"
-
-// Test the expansion of the Language List.
-TEST(ChromeAcceptLanguageSettings, ExpandLanguageList) {
-  std::string output = chrome_accept_language_settings::ExpandLanguageList("");
-  EXPECT_EQ("", output);
-
-  output = chrome_accept_language_settings::ExpandLanguageList("en-US");
-  EXPECT_EQ("en-US,en", output);
-
-  output = chrome_accept_language_settings::ExpandLanguageList("fr");
-  EXPECT_EQ("fr", output);
-
-  // The base language is added after all regional codes...
-  output = chrome_accept_language_settings::ExpandLanguageList("en-US,en-CA");
-  EXPECT_EQ("en-US,en-CA,en", output);
-
-  // ... but before other language families.
-  output =
-      chrome_accept_language_settings::ExpandLanguageList("en-US,en-CA,fr");
-  EXPECT_EQ("en-US,en-CA,en,fr", output);
-
-  output = chrome_accept_language_settings::ExpandLanguageList(
-      "en-US,en-CA,fr,en-AU");
-  EXPECT_EQ("en-US,en-CA,en,fr,en-AU", output);
-
-  output =
-      chrome_accept_language_settings::ExpandLanguageList("en-US,en-CA,fr-CA");
-  EXPECT_EQ("en-US,en-CA,en,fr-CA,fr", output);
-
-  // Add a base language even if it's already in the list.
-  output = chrome_accept_language_settings::ExpandLanguageList(
-      "en-US,fr-CA,it,fr,es-AR,it-IT");
-  EXPECT_EQ("en-US,en,fr-CA,fr,it,es-AR,es,it-IT", output);
-}

chrome/browser/net/profile_network_context_service.cc

@@ -13,13 +13,13 @@
 #include "build/build_config.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/content_settings/host_content_settings_map_factory.h"
-#include "chrome/browser/net/chrome_accept_language_settings.h"
 #include "chrome/browser/net/spdyproxy/data_reduction_proxy_chrome_settings.h"
 #include "chrome/browser/net/spdyproxy/data_reduction_proxy_chrome_settings_factory.h"
 #include "chrome/browser/net/system_network_context_manager.h"
 #include "chrome/browser/profiles/profile.h"
 #include "chrome/common/chrome_constants.h"
 #include "chrome/common/chrome_content_client.h"
+#include "chrome/common/chrome_features.h"
 #include "chrome/common/chrome_paths_internal.h"
 #include "chrome/common/pref_names.h"
 #include "components/certificate_transparency/pref_names.h"
@@ -35,6 +35,7 @@
 #include "content/public/common/service_names.mojom.h"
 #include "content/public/common/url_constants.h"
 #include "mojo/public/cpp/bindings/associated_interface_ptr.h"
+#include "net/http/http_util.h"
 #include "net/net_buildflags.h"
 #include "services/network/public/cpp/features.h"
 
@@ -53,6 +54,14 @@ std::vector<std::string> TranslateStringArray(const base::ListValue* list) {
   return strings;
 }
 
+std::string ComputeAcceptLanguageFromPref(const std::string& language_pref) {
+  std::string accept_languages_str =
+      base::FeatureList::IsEnabled(features::kUseNewAcceptLanguageHeader)
+          ? net::HttpUtil::ExpandLanguageList(language_pref)
+          : language_pref;
+  return net::HttpUtil::GenerateAcceptLanguageHeader(accept_languages_str);
+}
+
 }  // namespace
 
 ProfileNetworkContextService::ProfileNetworkContextService(Profile* profile)
@@ -218,8 +227,7 @@ void ProfileNetworkContextService::UpdateBlockThirdPartyCookies() {
 }
 
 std::string ProfileNetworkContextService::ComputeAcceptLanguage() const {
-  return chrome_accept_language_settings::ComputeAcceptLanguageFromPref(
-      pref_accept_language_.GetValue());
+  return ComputeAcceptLanguageFromPref(pref_accept_language_.GetValue());
 }
 
 void ProfileNetworkContextService::UpdateReferrersEnabled() {

chrome/test/BUILD.gn

@@ -2500,7 +2500,6 @@ test("unit_tests") {
     "../browser/metrics/thread_watcher_android_unittest.cc",
     "../browser/metrics/thread_watcher_unittest.cc",
     "../browser/navigation_predictor/navigation_predictor_unittest.cc",
-    "../browser/net/chrome_accept_language_settings_unittest.cc",
     "../browser/net/chrome_network_delegate_unittest.cc",
     "../browser/net/dns_probe_runner_unittest.cc",
     "../browser/net/dns_probe_service_unittest.cc",

net/http/http_util.cc

@@ -13,6 +13,7 @@
 #include "base/stl_util.h"
 #include "base/strings/string_number_conversions.h"
 #include "base/strings/string_piece.h"
+#include "base/strings/string_split.h"
 #include "base/strings/string_tokenizer.h"
 #include "base/strings/string_util.h"
 #include "base/strings/stringprintf.h"
@@ -34,6 +35,46 @@ void TrimLWSImplementation(ConstIterator* begin, ConstIterator* end) {
     --(*end);
 }
 
+// Helper class that builds the list of languages for the Accept-Language
+// headers.
+// The output is a comma-separated list of languages as string.
+// Duplicates are removed.
+class AcceptLanguageBuilder {
+ public:
+  // Adds a language to the string.
+  // Duplicates are ignored.
+  void AddLanguageCode(const std::string& language) {
+    // No Q score supported, only supports ASCII.
+    DCHECK_EQ(std::string::npos, language.find_first_of("; "));
+    DCHECK(base::IsStringASCII(language));
+    if (seen_.find(language) == seen_.end()) {
+      if (str_.empty()) {
+        base::StringAppendF(&str_, "%s", language.c_str());
+      } else {
+        base::StringAppendF(&str_, ",%s", language.c_str());
+      }
+      seen_.insert(language);
+    }
+  }
+
+  // Returns the string constructed up to this point.
+  std::string GetString() const { return str_; }
+
+ private:
+  // The string that contains the list of languages, comma-separated.
+  std::string str_;
+  // Set the remove duplicates.
+  std::unordered_set<std::string> seen_;
+};
+
+// Extract the base language code from a language code.
+// If there is no '-' in the code, the original code is returned.
+std::string GetBaseLanguageCode(const std::string& language_code) {
+  const std::vector<std::string> tokens = base::SplitString(
+      language_code, "-", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
+  return tokens.empty() ? "" : tokens[0];
+}
+
 }  // namespace
 
 // HttpUtil -------------------------------------------------------------------
@@ -773,6 +814,34 @@ std::string HttpUtil::ConvertHeadersBackToHTTPResponse(const std::string& str) {
   return disassembled_headers;
 }
 
+std::string HttpUtil::ExpandLanguageList(const std::string& language_prefs) {
+  const std::vector<std::string> languages = base::SplitString(
+      language_prefs, ",", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
+
+  if (languages.empty())
+    return "";
+
+  AcceptLanguageBuilder builder;
+
+  const int size = languages.size();
+  for (int i = 0; i < size; ++i) {
+    const std::string& language = languages[i];
+    builder.AddLanguageCode(language);
+
+    // Extract the base language
+    const std::string& base_language = GetBaseLanguageCode(language);
+
+    // Look ahead and add the base language if the next language is not part
+    // of the same family.
+    const int j = i + 1;
+    if (j >= size || GetBaseLanguageCode(languages[j]) != base_language) {
+      builder.AddLanguageCode(base_language);
+    }
+  }
+
+  return builder.GetString();
+}
+
 // TODO(jungshik): This function assumes that the input is a comma separated
 // list without any whitespace. As long as it comes from the preference and
 // a user does not manually edit the preference file, it's the case. Still,

net/http/http_util.h

@@ -204,6 +204,14 @@ class NET_EXPORT HttpUtil {
   // consists of status line and then one line for each header.
   static std::string ConvertHeadersBackToHTTPResponse(const std::string& str);
 
+  // Given a comma separated ordered list of language codes, return an expanded
+  // list by adding the base language from language-region pair if it doesn't
+  // already exist. This increases the chances of language matching in many
+  // cases as explained at this w3c doc:
+  // https://www.w3.org/International/questions/qa-lang-priorities#langtagdetail
+  // Note that we do not support Q values (e.g. ;q=0.9) in |language_prefs|.
+  static std::string ExpandLanguageList(const std::string& language_prefs);
+
   // Given a comma separated ordered list of language codes, return
   // the list with a qvalue appended to each language.
   // The way qvalues are assigned is rather simple. The qvalue

net/http/http_util_unittest.cc

@@ -1570,4 +1570,28 @@ TEST(HttpUtilTest, ParseContentEncoding) {
   }
 }
 
+// Test the expansion of the Language List.
+TEST(HttpUtilTest, ExpandLanguageList) {
+  EXPECT_EQ("", HttpUtil::ExpandLanguageList(""));
+  EXPECT_EQ("en-US,en", HttpUtil::ExpandLanguageList("en-US"));
+  EXPECT_EQ("fr", HttpUtil::ExpandLanguageList("fr"));
+
+  // The base language is added after all regional codes...
+  EXPECT_EQ("en-US,en-CA,en", HttpUtil::ExpandLanguageList("en-US,en-CA"));
+
+  // ... but before other language families.
+  EXPECT_EQ("en-US,en-CA,en,fr",
+            HttpUtil::ExpandLanguageList("en-US,en-CA,fr"));
+  EXPECT_EQ("en-US,en-CA,en,fr,en-AU",
+            HttpUtil::ExpandLanguageList("en-US,en-CA,fr,en-AU"));
+  EXPECT_EQ("en-US,en-CA,en,fr-CA,fr",
+            HttpUtil::ExpandLanguageList("en-US,en-CA,fr-CA"));
+
+  // Add a base language even if it's already in the list.
+  EXPECT_EQ("en-US,en,fr-CA,fr,it,es-AR,es,it-IT",
+            HttpUtil::ExpandLanguageList("en-US,fr-CA,it,fr,es-AR,it-IT"));
+  // Trims a whitespace.
+  EXPECT_EQ("en-US,en,fr", HttpUtil::ExpandLanguageList("en-US, fr"));
+}
+
 }  // namespace net