Language detection: dynamic content support.

This change adds a new language detection feature which is behind its
own feature flag to keep it separate from the existing language
detection support.

While the feature flag is off this change should not be user visible.

The earlier version of language detection only supported static content
- that is page content which was present at initial load.
This new feature supports dynamic content - that is page content loaded
after the initial page load.

R=dmazzoni,akihiroota,aboxhall

Bug: 889370
Change-Id: I98154c3d7a80f3f1a6bbf5bc6b29dc2088e01c41
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1697741
Commit-Queue: Chris Hall <chrishall@chromium.org>
Reviewed-by: Alice Boxhall <aboxhall@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#732298}

content/browser/accessibility/browser_accessibility_manager.cc

@@ -462,10 +462,16 @@ bool BrowserAccessibilityManager::OnAccessibilityEvents(
     // Some screen readers need a focus event in order to work properly.
     FireFocusEventsIfNeeded();
 
-    // Also, perform the initial run of language detection.
-    // TODO(chrishall): we will want to run this more often for dynamic pages.
-    tree_->language_detection_manager->DetectLanguages(tree_->root());
-    tree_->language_detection_manager->LabelLanguages(tree_->root());
+    // Perform the initial run of language detection.
+    tree_->language_detection_manager->DetectLanguages();
+    tree_->language_detection_manager->LabelLanguages();
+
+    // After initial language detection, enable language detection for future
+    // content updates in order to support dynamic content changes.
+    //
+    // If the LanguageDetectionDynamic feature flag is not enabled then this
+    // is a no-op.
+    tree_->language_detection_manager->RegisterLanguageDetectionObserver();
   }
 
   // Allow derived classes to do event post-processing.

extensions/renderer/api/automation/automation_ax_tree_wrapper.cc

@@ -323,13 +323,19 @@ bool AutomationAXTreeWrapper::OnAccessibilityEvents(
   // Currently language detection only runs once for initial load complete, any
   // content loaded after this will not have language detection performed for
   // it.
-  //
-  // TODO(chrishall): We may want to run this more often for dynamic content.
   for (const auto& targeted_event : event_generator_) {
     if (targeted_event.event_params.event ==
         ui::AXEventGenerator::Event::LOAD_COMPLETE) {
-      tree_.language_detection_manager->DetectLanguages(tree_.root());
-      tree_.language_detection_manager->LabelLanguages(tree_.root());
+      tree_.language_detection_manager->DetectLanguages();
+      tree_.language_detection_manager->LabelLanguages();
+
+      // After initial language detection, enable language detection for future
+      // content updates in order to support dynamic content changes.
+      //
+      // If the LanguageDetectionDynamic feature flag is not enabled then this
+      // is a no-op.
+      tree_.language_detection_manager->RegisterLanguageDetectionObserver();
+
       break;
     }
   }

ui/accessibility/ax_language_detection.cc

@@ -96,20 +96,33 @@ void AXLanguageInfoStats::GenerateTopResults() {
   top_results_valid_ = true;
 }
 
-AXLanguageDetectionManager::AXLanguageDetectionManager()
+AXLanguageDetectionManager::AXLanguageDetectionManager(AXTree* tree)
     : short_text_language_identifier_(kShortTextIdentifierMinByteLength,
-                                      kShortTextIdentifierMaxByteLength) {}
+                                      kShortTextIdentifierMaxByteLength),
+      tree_(tree) {}
+
 AXLanguageDetectionManager::~AXLanguageDetectionManager() = default;
 
-// Detect languages for a subtree rooted at the given subtree_root.
-void AXLanguageDetectionManager::DetectLanguages(AXNode* subtree_root) {
+void AXLanguageDetectionManager::RegisterLanguageDetectionObserver() {
+  // If the dynamic feature flag is not enabled then do nothing.
+  if (!::switches::
+          IsExperimentalAccessibilityLanguageDetectionDynamicEnabled()) {
+    return;
+  }
+
+  // Construct our new Observer as requested.
+  // If there is already an Observer on this Manager then this will destroy it.
+  language_detection_observer_.reset(new AXLanguageDetectionObserver(tree_));
+}
+
+// Detect languages for each node.
+void AXLanguageDetectionManager::DetectLanguages() {
   TRACE_EVENT0("accessibility", "AXLanguageInfo::DetectLanguages");
-  DCHECK(subtree_root);
   if (!::switches::IsExperimentalAccessibilityLanguageDetectionEnabled()) {
     return;
   }
 
-  DetectLanguagesForSubtree(subtree_root);
+  DetectLanguagesForSubtree(tree_->root());
 }
 
 // Detect languages for a subtree rooted at the given subtree_root.
@@ -165,29 +178,32 @@ void AXLanguageDetectionManager::DetectLanguagesForNode(AXNode* node) {
   if (reliable_results.size()) {
     AXLanguageInfo* lang_info = node->GetLanguageInfo();
     if (lang_info) {
+      // Clear previously detected and labelled languages.
       lang_info->detected_languages.clear();
+      lang_info->language.clear();
     } else {
       node->SetLanguageInfo(std::make_unique<AXLanguageInfo>());
       lang_info = node->GetLanguageInfo();
     }
 
+    // Keep these results.
     lang_info->detected_languages = std::move(reliable_results);
+
+    // Update statistics to take these results into account.
     lang_info_stats_.Add(lang_info->detected_languages);
   }
 }
 
-// Label languages for each node in the subtree rooted at the given
-// subtree_root. This relies on DetectLanguages having already been run.
-void AXLanguageDetectionManager::LabelLanguages(AXNode* subtree_root) {
+// Label languages for each node. This relies on DetectLanguages having already
+// been run.
+void AXLanguageDetectionManager::LabelLanguages() {
   TRACE_EVENT0("accessibility", "AXLanguageInfo::LabelLanguages");
 
-  DCHECK(subtree_root);
-
   if (!::switches::IsExperimentalAccessibilityLanguageDetectionEnabled()) {
     return;
   }
 
-  LabelLanguagesForSubtree(subtree_root);
+  LabelLanguagesForSubtree(tree_->root());
 }
 
 // Label languages for each node in the subtree rooted at the given
@@ -295,4 +311,63 @@ AXLanguageDetectionManager::GetLanguageAnnotationForStringAttribute(
   return language_annotation;
 }
 
+AXLanguageDetectionObserver::AXLanguageDetectionObserver(AXTree* tree)
+    : tree_(tree) {
+  // We expect the feature flag to have be checked before this Observer is
+  // constructed, this should have been checked by
+  // RegisterLanguageDetectionObserver.
+  DCHECK(
+      ::switches::IsExperimentalAccessibilityLanguageDetectionDynamicEnabled());
+
+  tree_->AddObserver(this);
+}
+
+AXLanguageDetectionObserver::~AXLanguageDetectionObserver() {
+  tree_->RemoveObserver(this);
+}
+
+void AXLanguageDetectionObserver::OnAtomicUpdateFinished(
+    ui::AXTree* tree,
+    bool root_changed,
+    const std::vector<Change>& changes) {
+  // TODO(chrishall): We likely want to re-consider updating or resetting
+  // AXLanguageInfoStats over time to better support detection on long running
+  // pages.
+
+  // TODO(chrishall): To support pruning deleted node data from stats we should
+  // consider implementing OnNodeWillBeDeleted. Other options available include:
+  // 1) move lang info from AXNode into a map on AXTree so that we can fetch
+  //    based on id in here
+  // 2) AXLanguageInfo destructor could remove itself
+
+  // TODO(chrishall): Possible optimisation: only run detect/label for certain
+  // change.type(s)), at least NODE_CREATED, NODE_CHANGED, and SUBTREE_CREATED.
+
+  DCHECK(tree->language_detection_manager);
+
+  // Perform Detect and Label for each node changed or created.
+  // We currently only consider kStaticText for detection.
+  //
+  // Note that language inheritance is now handled by AXNode::GetLanguage.
+  //
+  // Note that since Label no longer handles language inheritance, we only need
+  // to call Label and Detect on the nodes that changed and don't need to
+  // recurse.
+  //
+  // We do this in two passes because Detect updates page level statistics which
+  // are later used by Label in order to make more accurate decisions.
+
+  for (auto& change : changes) {
+    if (change.node->data().role == ax::mojom::Role::kStaticText) {
+      tree->language_detection_manager->DetectLanguagesForNode(change.node);
+    }
+  }
+
+  for (auto& change : changes) {
+    if (change.node->data().role == ax::mojom::Role::kStaticText) {
+      tree->language_detection_manager->LabelLanguagesForNode(change.node);
+    }
+  }
+}
+
 }  // namespace ui

ui/accessibility/ax_language_detection.h

@@ -5,6 +5,7 @@
 #ifndef UI_ACCESSIBILITY_AX_LANGUAGE_DETECTION_H_
 #define UI_ACCESSIBILITY_AX_LANGUAGE_DETECTION_H_
 
+#include <memory>
 #include <string>
 #include <unordered_map>
 #include <utility>
@@ -14,6 +15,7 @@
 #include "third_party/cld_3/src/src/nnet_language_identifier.h"
 #include "ui/accessibility/ax_enums.mojom-forward.h"
 #include "ui/accessibility/ax_export.h"
+#include "ui/accessibility/ax_tree_observer.h"
 
 namespace ui {
 
@@ -139,24 +141,57 @@ class AX_EXPORT AXLanguageInfoStats {
   DISALLOW_COPY_AND_ASSIGN(AXLanguageInfoStats);
 };
 
+// AXLanguageDetectionObserver is registered as a change observer on an AXTree
+// and will run language detection after each update to the tree.
+//
+// We have kept this observer separate from the AXLanguageDetectionManager as we
+// are aiming to launch language detection in two phases and wanted to try keep
+// the code paths somewhat separate.
+//
+// TODO(chrishall): After both features have launched we could consider merging
+// AXLanguageDetectionObserver into AXLanguageDetectionManager.
+//
+// TODO(chrishall): Investigate the cost of using AXTreeObserver, given that it
+// has many empty virtual methods which are called for every AXTree change and
+// we are only currently interested in OnAtomicUpdateFinished.
+class AX_EXPORT AXLanguageDetectionObserver : public ui::AXTreeObserver {
+ public:
+  // Observer constructor will register itself with the provided AXTree.
+  AXLanguageDetectionObserver(AXTree* tree);
+
+  // Observer destructor will remove itself as an observer from the AXTree.
+  ~AXLanguageDetectionObserver() override;
+
+ private:
+  void OnAtomicUpdateFinished(ui::AXTree* tree,
+                              bool root_changed,
+                              const std::vector<Change>& changes) override;
+
+  // Non-owning pointer to AXTree, used to de-register observer on destruction.
+  AXTree* const tree_;
+
+  DISALLOW_COPY_AND_ASSIGN(AXLanguageDetectionObserver);
+};
+
 // AXLanguageDetectionManager manages all of the context needed for language
 // detection within an AXTree.
 class AX_EXPORT AXLanguageDetectionManager {
  public:
-  AXLanguageDetectionManager();
+  // Construct an AXLanguageDetectionManager for the specified tree.
+  AXLanguageDetectionManager(AXTree* tree);
   ~AXLanguageDetectionManager();
 
-  // Detect languages for each node in the subtree rooted at the given node.
+  // Detect languages for each node in the tree managed by this manager.
   // This is the first pass in detection and labelling.
   // This only detects the language, it does not label it, for that see
   //  LabelLanguageForSubtree.
-  void DetectLanguages(AXNode* subtree_root);
+  void DetectLanguages();
 
-  // Label languages for each node in the subtree rooted at the given node.
+  // Label languages for each node in the tree manager by this manager.
   // This is the second pass in detection and labelling.
   // This will label the language, but relies on the earlier detection phase
   // having already completed.
-  void LabelLanguages(AXNode* subtree_root);
+  void LabelLanguages();
 
   // Sub-node language detection for a given string attribute.
   // For example, if a node has name: "My name is Fred", then calling
@@ -166,8 +201,14 @@ class AX_EXPORT AXLanguageDetectionManager {
       const AXNode& node,
       ax::mojom::StringAttribute attr);
 
+  // Construct and register a dynamic content change observer for this manager.
+  void RegisterLanguageDetectionObserver();
+
  private:
-  AXLanguageInfoStats lang_info_stats_;
+  friend class AXLanguageDetectionObserver;
+
+  // Allow access from a fixture only used in testing.
+  friend class AXLanguageDetectionTestFixture;
 
   // Perform detection for subtree rooted at subtree_root.
   void DetectLanguagesForSubtree(AXNode* subtree_root);
@@ -188,6 +229,14 @@ class AX_EXPORT AXLanguageDetectionManager {
   // of shorter text (e.g. one character).
   chrome_lang_id::NNetLanguageIdentifier short_text_language_identifier_;
 
+  // The observer to support dynamic content language detection.
+  std::unique_ptr<AXLanguageDetectionObserver> language_detection_observer_;
+
+  // Non-owning back pointer to the tree which owns this manager.
+  AXTree* tree_;
+
+  AXLanguageInfoStats lang_info_stats_;
+
   DISALLOW_COPY_AND_ASSIGN(AXLanguageDetectionManager);
 };
 

ui/accessibility/ax_tree.cc

@@ -577,13 +577,15 @@ AXTree::AXTree() {
   // TODO(chrishall): do we want to initialize all the time, on demand, or only
   //                  when feature flag is set?
   DCHECK(!language_detection_manager);
-  language_detection_manager = std::make_unique<AXLanguageDetectionManager>();
+  language_detection_manager =
+      std::make_unique<AXLanguageDetectionManager>(this);
 }
 
 AXTree::AXTree(const AXTreeUpdate& initial_state) {
   CHECK(Unserialize(initial_state)) << error();
   DCHECK(!language_detection_manager);
-  language_detection_manager = std::make_unique<AXLanguageDetectionManager>();
+  language_detection_manager =
+      std::make_unique<AXLanguageDetectionManager>(this);
 }
 
 AXTree::~AXTree() {