Quick Answers: Enhance language detection

Currently we detect the language only using the surrounding text.
Check user selected text first and fall back to the surrounding text if
the confidence level is no high enough.

Bug: b/172374961
Test: Run existing tests
Change-Id: Ifbcd820828aa7628994b90b756bcd2c9797f30c5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2528646Reviewed-by: Xiaohui Chen <xiaohuic@chromium.org>
Reviewed-by: Xiyuan Xia <xiyuan@chromium.org>
Reviewed-by: Jeroen Dhollander <jeroendh@chromium.org>
Commit-Queue: Yue Li <updowndota@chromium.org>
Cr-Commit-Position: refs/heads/master@{#827062}

chromeos/components/quick_answers/BUILD.gn

@@ -32,6 +32,8 @@ source_set("quick_answers") {
     "translation_result_loader.h",
     "understanding/intent_generator.cc",
     "understanding/intent_generator.h",
+    "utils/language_detector.cc",
+    "utils/language_detector.h",
     "utils/quick_answers_metrics.cc",
     "utils/quick_answers_metrics.h",
     "utils/quick_answers_utils.cc",

chromeos/components/quick_answers/understanding/intent_generator.cc

@@ -144,11 +144,11 @@ void IntentGenerator::LoadModelCallback(const QuickAnswersRequest& request,
   }
 
   if (text_classifier_) {
-    text_classifier_->FindLanguages(
-        !request.context.surrounding_text.empty()
-            ? request.context.surrounding_text
-            : request.selected_text,
-        base::BindOnce(&IntentGenerator::FindLanguagesCallback,
+    language_detector_ =
+        std::make_unique<LanguageDetector>(text_classifier_.get());
+    language_detector_->DetectLanguage(
+        request.context.surrounding_text, request.selected_text,
+        base::BindOnce(&IntentGenerator::LanguageDetectorCallback,
                        weak_factory_.GetWeakPtr(), request));
   }
 }
@@ -180,14 +180,14 @@ void IntentGenerator::AnnotationCallback(
       .Run(IntentInfo(request.selected_text, IntentType::kUnknown));
 }
 
-void IntentGenerator::FindLanguagesCallback(
+void IntentGenerator::LanguageDetectorCallback(
     const QuickAnswersRequest& request,
-    std::vector<machine_learning::mojom::TextLanguagePtr> languages) {
-  // TODO(b/150034512): Take confidence level into consideration.
-  if (!languages.empty() &&
+    base::Optional<std::string> detected_locale) {
+  language_detector_.reset();
+  if (detected_locale.has_value() &&
       !request.context.device_properties.language.empty() &&
-      languages.front()->locale != request.context.device_properties.language) {
-    MaybeGenerateTranslationIntent(request, languages.front()->locale);
+      detected_locale.value() != request.context.device_properties.language) {
+    MaybeGenerateTranslationIntent(request, detected_locale.value());
     return;
   }
 

chromeos/components/quick_answers/understanding/intent_generator.h

@@ -9,6 +9,7 @@
 #include <string>
 
 #include "base/callback.h"
+#include "chromeos/components/quick_answers/utils/language_detector.h"
 #include "chromeos/services/machine_learning/public/mojom/machine_learning_service.mojom.h"
 #include "chromeos/services/machine_learning/public/mojom/text_classifier.mojom.h"
 #include "mojo/public/cpp/bindings/remote.h"
@@ -50,9 +51,8 @@ class IntentGenerator {
   void AnnotationCallback(
       const QuickAnswersRequest& request,
       std::vector<machine_learning::mojom::TextAnnotationPtr> annotations);
-  void FindLanguagesCallback(
-      const QuickAnswersRequest& request,
-      std::vector<machine_learning::mojom::TextLanguagePtr> languages);
+  void LanguageDetectorCallback(const QuickAnswersRequest& request,
+                                base::Optional<std::string> detected_locale);
 
   void MaybeGenerateTranslationIntent(const QuickAnswersRequest& request,
                                       const std::string& detected_locale);
@@ -60,6 +60,7 @@ class IntentGenerator {
   IntentGeneratorCallback complete_callback_;
   mojo::Remote<::chromeos::machine_learning::mojom::TextClassifier>
       text_classifier_;
+  std::unique_ptr<LanguageDetector> language_detector_;
 
   base::WeakPtrFactory<IntentGenerator> weak_factory_{this};
 };

chromeos/components/quick_answers/utils/language_detector.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chromeos/components/quick_answers/utils/language_detector.h"
+
+#include "base/callback.h"
+
+namespace chromeos {
+namespace quick_answers {
+namespace {
+
+// TODO(b/173143583): Make the thresholds finch params.
+constexpr float kSelectedTextConfidenceThreshold = 0.7;
+constexpr float kSurroundingTextConfidenceThreshold = 0.9;
+
+base::Optional<std::string> GetLanguageWithConfidence(
+    const std::vector<machine_learning::mojom::TextLanguagePtr>& languages,
+    float confidence_threshold) {
+  // The languages are sorted according to the confidence score, from the
+  // highest to the lowest (according to the mojom method documentation).
+  if (!languages.empty() &&
+      languages.front()->confidence > confidence_threshold) {
+    return languages.front()->locale;
+  }
+  return base::nullopt;
+}
+
+}  // namespace
+
+LanguageDetector::LanguageDetector(
+    chromeos::machine_learning::mojom::TextClassifier* text_classifier)
+    : text_classifier_(text_classifier) {}
+
+LanguageDetector::~LanguageDetector() = default;
+
+void LanguageDetector::DetectLanguage(const std::string& surrounding_text,
+                                      const std::string& selected_text,
+                                      DetectLanguageCallback callback) {
+  text_classifier_->FindLanguages(
+      selected_text,
+      base::BindOnce(&LanguageDetector::FindLanguagesForSelectedTextCallback,
+                     weak_factory_.GetWeakPtr(), surrounding_text,
+                     std::move(callback)));
+}
+
+void LanguageDetector::FindLanguagesForSelectedTextCallback(
+    const std::string& surrounding_text,
+    DetectLanguageCallback callback,
+    std::vector<machine_learning::mojom::TextLanguagePtr> languages) {
+  auto locale = GetLanguageWithConfidence(std::move(languages),
+                                          kSelectedTextConfidenceThreshold);
+  if (locale.has_value()) {
+    std::move(callback).Run(std::move(locale));
+    return;
+  }
+
+  // If find language failed or the confidence level is too low, fall back to
+  // find language for the surrounding text.
+  text_classifier_->FindLanguages(
+      surrounding_text,
+      base::BindOnce(&LanguageDetector::FindLanguagesForSurroundingTextCallback,
+                     weak_factory_.GetWeakPtr(), std::move(callback)));
+}
+
+void LanguageDetector::FindLanguagesForSurroundingTextCallback(
+    DetectLanguageCallback callback,
+    std::vector<machine_learning::mojom::TextLanguagePtr> languages) {
+  auto locale =
+      GetLanguageWithConfidence(languages, kSurroundingTextConfidenceThreshold);
+
+  std::move(callback).Run(std::move(locale));
+}
+
+}  // namespace quick_answers
+}  // namespace chromeos

chromeos/components/quick_answers/utils/language_detector.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef CHROMEOS_COMPONENTS_QUICK_ANSWERS_UTILS_LANGUAGE_DETECTOR_H_
+#define CHROMEOS_COMPONENTS_QUICK_ANSWERS_UTILS_LANGUAGE_DETECTOR_H_
+
+#include <memory>
+#include <string>
+
+#include "base/callback_forward.h"
+#include "base/memory/weak_ptr.h"
+#include "base/optional.h"
+#include "chromeos/services/machine_learning/public/mojom/text_classifier.mojom.h"
+
+namespace chromeos {
+namespace quick_answers {
+
+// Utility class for language detection.
+// TODO(b/172992698): Add test support.
+class LanguageDetector {
+ public:
+  using DetectLanguageCallback =
+      base::OnceCallback<void(base::Optional<std::string>)>;
+
+  explicit LanguageDetector(
+      chromeos::machine_learning::mojom::TextClassifier* text_classifier);
+
+  LanguageDetector(const LanguageDetector&) = delete;
+  LanguageDetector& operator=(const LanguageDetector&) = delete;
+
+  ~LanguageDetector();
+
+  // Returns language code of the specified |selected_text|.
+  // Fall back to language code of |surrounding_text| if the confidence level is
+  // not high enough.
+  // Returns no value if no language can be detected.
+  void DetectLanguage(const std::string& surrounding_text,
+                      const std::string& selected_text,
+                      DetectLanguageCallback callback);
+
+ private:
+  void FindLanguagesForSelectedTextCallback(
+      const std::string& surrounding_text,
+      DetectLanguageCallback callback,
+      std::vector<machine_learning::mojom::TextLanguagePtr> languages);
+
+  void FindLanguagesForSurroundingTextCallback(
+      DetectLanguageCallback callback,
+      std::vector<machine_learning::mojom::TextLanguagePtr> languages);
+
+  // Owned by IntentGenerator.
+  chromeos::machine_learning::mojom::TextClassifier* text_classifier_ = nullptr;
+
+  base::WeakPtrFactory<LanguageDetector> weak_factory_{this};
+};
+
+}  // namespace quick_answers
+}  // namespace chromeos
+
+#endif  // CHROMEOS_COMPONENTS_QUICK_ANSWERS_UTILS_LANGUAGE_DETECTOR_H_