Android accessibility: Expose SuggestionSpans for spelling errors

TalkBack doesn't yet consume these yet, but this matches the same way
native Android TextViews expose misspelled words, whether from typing,
or from dictation with multiple suggestions.

Bug: 524654
Change-Id: I3a78b78f12a5a9c67b15718422d7c3c6cc524376
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1881695
Commit-Queue: Dominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: Nektarios Paisios <nektar@chromium.org>
Reviewed-by: Jinsuk Kim <jinsukkim@chromium.org>
Cr-Commit-Position: refs/heads/master@{#715098}

content/browser/accessibility/browser_accessibility.h

@@ -573,6 +573,11 @@ class CONTENT_EXPORT BrowserAccessibility : public ui::AXPlatformNodeDelegate {
   // Does need to be called by subclasses such as BrowserAccessibilityAndroid.
   const ui::AXUniqueId& GetUniqueId() const override;
 
+  // Returns a text attribute map indicating the offsets in the text of a leaf
+  // object, such as a text field or static text, where spelling and grammar
+  // errors are present.
+  ui::TextAttributeMap GetSpellingAndGrammarAttributes() const;
+
  private:
   // Return the bounds after converting from this node's coordinate system
   // (which is relative to its nearest scrollable ancestor) to the coordinate
@@ -618,11 +623,6 @@ class CONTENT_EXPORT BrowserAccessibility : public ui::AXPlatformNodeDelegate {
   // If the node has a child tree, get the root node.
   BrowserAccessibility* PlatformGetRootOfChildTree() const;
 
-  // Returns a text attribute map indicating the offsets in the text of a leaf
-  // object, such as a text field or static text, where spelling and grammar
-  // errors are present.
-  ui::TextAttributeMap GetSpellingAndGrammarAttributes() const;
-
   // Given a set of map of spelling text attributes and a start offset, merge
   // them into the given map of existing text attributes. Merges the given
   // spelling attributes, i.e. document marker information, into the given text

content/browser/accessibility/browser_accessibility_android.cc

@@ -1707,6 +1707,69 @@ base::string16 BrowserAccessibilityAndroid::GetTargetUrl() const {
   return {};
 }
 
+void BrowserAccessibilityAndroid::GetSuggestions(
+    std::vector<int>* suggestion_starts,
+    std::vector<int>* suggestion_ends) const {
+  DCHECK(suggestion_starts);
+  DCHECK(suggestion_ends);
+
+  if (!IsRichTextField() && !IsPlainTextField())
+    return;
+
+  // TODO: using FindTextOnlyObjectsInRange or NextInTreeOrder doesn't
+  // work because Android's PlatformIsLeafIncludingIgnored implementation
+  // deliberately excludes a lot of nodes. We need a version of
+  // FindTextOnlyObjectsInRange and/or NextInTreeOrder that only walk
+  // the internal tree.
+  BrowserAccessibility* node = InternalGetFirstChild();
+  int start_offset = 0;
+  while (node && node != this) {
+    if (node->IsTextOnlyObject()) {
+      const std::vector<int32_t>& marker_types =
+          node->GetData().GetIntListAttribute(
+              ax::mojom::IntListAttribute::kMarkerTypes);
+      if (!marker_types.empty()) {
+        const std::vector<int>& marker_starts =
+            node->GetData().GetIntListAttribute(
+                ax::mojom::IntListAttribute::kMarkerStarts);
+        const std::vector<int>& marker_ends =
+            node->GetData().GetIntListAttribute(
+                ax::mojom::IntListAttribute::kMarkerEnds);
+
+        for (size_t i = 0; i < marker_types.size(); ++i) {
+          // On Android, both spelling errors and alternatives from dictation
+          // are both encoded as suggestions.
+          if (!(marker_types[i] &
+                static_cast<int32_t>(ax::mojom::MarkerType::kSuggestion))) {
+            continue;
+          }
+
+          int suggestion_start = start_offset + marker_starts[i];
+          int suggestion_end = start_offset + marker_ends[i];
+          suggestion_starts->push_back(suggestion_start);
+          suggestion_ends->push_back(suggestion_end);
+        }
+      }
+      start_offset += node->GetText().length();
+    }
+
+    // Implementation of NextInTreeOrder, but walking the internal tree.
+    if (node->InternalChildCount()) {
+      node = node->InternalGetFirstChild();
+    } else {
+      while (node && node != this) {
+        BrowserAccessibility* sibling = node->InternalGetNextSibling();
+        if (sibling) {
+          node = sibling;
+          break;
+        }
+
+        node = node->InternalGetParent();
+      }
+    }
+  }
+}
+
 bool BrowserAccessibilityAndroid::HasNonEmptyValue() const {
   return IsEditableText() && !GetValue().empty();
 }

content/browser/accessibility/browser_accessibility_android.h

@@ -157,6 +157,12 @@ class CONTENT_EXPORT BrowserAccessibilityAndroid : public BrowserAccessibility {
   // Return the target of a link or the source of an image.
   base::string16 GetTargetUrl() const;
 
+  // On Android, spelling errors are returned as "suggestions". Retreive
+  // all of the suggestions for a given text field as vectors of start
+  // and end offsets.
+  void GetSuggestions(std::vector<int>* suggestion_starts,
+                      std::vector<int>* suggestion_ends) const;
+
   // Used to keep track of when to stop reporting content_invalid.
   // Timer only applies if node has focus.
   void ResetContentInvalidTimer();

content/browser/accessibility/web_contents_accessibility_android.cc

@@ -697,13 +697,35 @@ jboolean WebContentsAccessibilityAndroid::PopulateAccessibilityNodeInfo(
   Java_WebContentsAccessibilityImpl_setAccessibilityNodeInfoClassName(
       env, obj, info,
       base::android::ConvertUTF8ToJavaString(env, node->GetClassName()));
+
+  ScopedJavaLocalRef<jintArray> suggestion_starts_java;
+  ScopedJavaLocalRef<jintArray> suggestion_ends_java;
+  ScopedJavaLocalRef<jobjectArray> suggestion_text_java;
+  std::vector<int> suggestion_starts;
+  std::vector<int> suggestion_ends;
+  node->GetSuggestions(&suggestion_starts, &suggestion_ends);
+  if (suggestion_starts.size() && suggestion_ends.size()) {
+    suggestion_starts_java = base::android::ToJavaIntArray(
+        env, suggestion_starts.data(), suggestion_starts.size());
+    suggestion_ends_java = base::android::ToJavaIntArray(
+        env, suggestion_ends.data(), suggestion_ends.size());
+
+    // Currently we don't retrieve the text of each suggestion, so
+    // store a blank string for now.
+    std::vector<std::string> suggestion_text(suggestion_starts.size());
+    suggestion_text_java =
+        base::android::ToJavaArrayOfStrings(env, suggestion_text);
+  }
+
   Java_WebContentsAccessibilityImpl_setAccessibilityNodeInfoText(
       env, obj, info,
       base::android::ConvertUTF16ToJavaString(env, node->GetInnerText()),
       node->IsLink(), node->IsEditableText(),
       base::android::ConvertUTF16ToJavaString(
           env, node->GetInheritedString16Attribute(
-                   ax::mojom::StringAttribute::kLanguage)));
+                   ax::mojom::StringAttribute::kLanguage)),
+      suggestion_starts_java, suggestion_ends_java, suggestion_text_java);
+
   base::string16 element_id;
   if (node->GetHtmlAttribute("id", &element_id)) {
     Java_WebContentsAccessibilityImpl_setAccessibilityNodeInfoViewIdResourceName(
@@ -1053,6 +1075,36 @@ jint WebContentsAccessibilityAndroid::GetTextLength(
   return text.size();
 }
 
+void WebContentsAccessibilityAndroid::AddSpellingErrorForTesting(
+    JNIEnv* env,
+    const JavaParamRef<jobject>& obj,
+    jint unique_id,
+    jint start_offset,
+    jint end_offset) {
+  BrowserAccessibility* node = GetAXFromUniqueID(unique_id);
+  CHECK(node);
+
+  while (node->GetRole() != ax::mojom::Role::kStaticText &&
+         node->InternalChildCount() > 0) {
+    node = node->InternalChildrenBegin().get();
+  }
+
+  CHECK(node->GetRole() == ax::mojom::Role::kStaticText);
+  base::string16 text = node->GetInnerText();
+  CHECK_LT(start_offset, static_cast<int>(text.size()));
+  CHECK_LE(end_offset, static_cast<int>(text.size()));
+
+  ui::AXNodeData data = node->GetData();
+  data.AddIntListAttribute(ax::mojom::IntListAttribute::kMarkerStarts,
+                           {start_offset});
+  data.AddIntListAttribute(ax::mojom::IntListAttribute::kMarkerEnds,
+                           {end_offset});
+  data.AddIntListAttribute(
+      ax::mojom::IntListAttribute::kMarkerTypes,
+      {static_cast<int>(ax::mojom::MarkerType::kSuggestion)});
+  node->node()->SetData(data);
+}
+
 jboolean WebContentsAccessibilityAndroid::PreviousAtGranularity(
     JNIEnv* env,
     const JavaParamRef<jobject>& obj,

content/browser/accessibility/web_contents_accessibility_android.h

@@ -202,6 +202,14 @@ class CONTENT_EXPORT WebContentsAccessibilityAndroid
                      const base::android::JavaParamRef<jobject>& obj,
                      jint id);
 
+  // Add a fake spelling error for testing spelling spannables.
+  void AddSpellingErrorForTesting(
+      JNIEnv* env,
+      const base::android::JavaParamRef<jobject>& obj,
+      jint id,
+      jint start_offset,
+      jint end_offset);
+
   // Request loading inline text boxes for a given node.
   void LoadInlineTextBoxes(JNIEnv* env,
                            const base::android::JavaParamRef<jobject>& obj,

content/public/android/java/src/org/chromium/content/browser/accessibility/LollipopWebContentsAccessibility.java

@@ -13,6 +13,7 @@ import android.content.ReceiverCallNotAllowedException;
 import android.os.Build;
 import android.text.SpannableString;
 import android.text.style.LocaleSpan;
+import android.text.style.SuggestionSpan;
 import android.util.SparseArray;
 import android.view.accessibility.AccessibilityEvent;
 import android.view.accessibility.AccessibilityNodeInfo;
@@ -141,8 +142,10 @@ public class LollipopWebContentsAccessibility extends KitKatWebContentsAccessibi
     }
 
     @Override
-    protected CharSequence computeText(String text, boolean annotateAsLink, String language) {
-        CharSequence charSequence = super.computeText(text, annotateAsLink, language);
+    protected CharSequence computeText(String text, boolean annotateAsLink, String language,
+            int[] suggestionStarts, int[] suggestionEnds, String[] suggestions) {
+        CharSequence charSequence = super.computeText(
+                text, annotateAsLink, language, suggestionStarts, suggestionEnds, suggestions);
         if (!language.isEmpty() && !language.equals(mSystemLanguageTag)) {
             SpannableString spannable;
             if (charSequence instanceof SpannableString) {
@@ -152,8 +155,33 @@ public class LollipopWebContentsAccessibility extends KitKatWebContentsAccessibi
             }
             Locale locale = Locale.forLanguageTag(language);
             spannable.setSpan(new LocaleSpan(locale), 0, spannable.length(), 0);
-            return spannable;
+            charSequence = spannable;
         }
+
+        if (suggestionStarts != null && suggestionStarts.length > 0) {
+            assert suggestionEnds != null;
+            assert suggestionEnds.length == suggestionStarts.length;
+            assert suggestions != null;
+            assert suggestions.length == suggestionStarts.length;
+
+            SpannableString spannable;
+            if (charSequence instanceof SpannableString) {
+                spannable = (SpannableString) charSequence;
+            } else {
+                spannable = new SpannableString(charSequence);
+            }
+
+            for (int i = 0; i < suggestionStarts.length; i++) {
+                String[] suggestionArray = new String[1];
+                suggestionArray[0] = suggestions[i];
+                int flags = SuggestionSpan.FLAG_MISSPELLED;
+                SuggestionSpan suggestionSpan =
+                        new SuggestionSpan(mContext, suggestionArray, flags);
+                spannable.setSpan(suggestionSpan, suggestionStarts[i], suggestionEnds[i], 0);
+            }
+            charSequence = spannable;
+        }
+
         return charSequence;
     }
 

content/public/android/java/src/org/chromium/content/browser/accessibility/WebContentsAccessibilityImpl.java

@@ -195,6 +195,13 @@ public class WebContentsAccessibilityImpl extends AccessibilityNodeProvider
         mAccessibilityEnabledForTesting = true;
     }
 
+    @VisibleForTesting
+    @Override
+    public void addSpellingErrorForTesting(int virtualViewId, int startOffset, int endOffset) {
+        WebContentsAccessibilityImplJni.get().addSpellingErrorForTesting(mNativeObj,
+                WebContentsAccessibilityImpl.this, virtualViewId, startOffset, endOffset);
+    }
+
     // WindowEventObserver
 
     @Override
@@ -1230,12 +1237,15 @@ public class WebContentsAccessibilityImpl extends AccessibilityNodeProvider
     @SuppressLint("NewApi")
     @CalledByNative
     private void setAccessibilityNodeInfoText(AccessibilityNodeInfo node, String text,
-            boolean annotateAsLink, boolean isEditableText, String language) {
-        CharSequence computedText = computeText(text, isEditableText, language);
+            boolean annotateAsLink, boolean isEditableText, String language, int[] suggestionStarts,
+            int[] suggestionEnds, String[] suggestions) {
+        CharSequence computedText = computeText(
+                text, isEditableText, language, suggestionStarts, suggestionEnds, suggestions);
         node.setText(computedText);
     }
 
-    protected CharSequence computeText(String text, boolean annotateAsLink, String language) {
+    protected CharSequence computeText(String text, boolean annotateAsLink, String language,
+            int[] suggestionStarts, int[] suggestionEnds, String[] suggestions) {
         if (annotateAsLink) {
             SpannableString spannable = new SpannableString(text);
             spannable.setSpan(new URLSpan(""), 0, spannable.length(), 0);
@@ -1613,5 +1623,7 @@ public class WebContentsAccessibilityImpl extends AccessibilityNodeProvider
                 WebContentsAccessibilityImpl caller, int id, int start, int len);
         int getTextLength(long nativeWebContentsAccessibilityAndroid,
                 WebContentsAccessibilityImpl caller, int id);
+        void addSpellingErrorForTesting(long nativeWebContentsAccessibilityAndroid,
+                WebContentsAccessibilityImpl caller, int id, int startOffset, int endOffset);
     }
 }

content/public/android/java/src/org/chromium/content_public/browser/WebContentsAccessibility.java

@@ -48,6 +48,12 @@ public interface WebContentsAccessibility {
     @VisibleForTesting
     void setAccessibilityEnabledForTesting();
 
+    /**
+     *  Add a spelling error.
+     */
+    @VisibleForTesting
+    void addSpellingErrorForTesting(int virtualViewId, int startOffset, int endOffset);
+
     /**
      * Attempts to perform an accessibility action on the web content.  If the accessibility action
      * cannot be processed, it returns {@code null}, allowing the caller to know to call the

content/public/android/javatests/src/org/chromium/content/browser/accessibility/WebContentsAccessibilityTest.java

@@ -15,6 +15,8 @@ import android.os.Build;
 import android.os.Bundle;
 import android.support.test.filters.MediumTest;
 import android.text.InputType;
+import android.text.Spannable;
+import android.text.style.SuggestionSpan;
 import android.view.View;
 import android.view.ViewGroup;
 import android.view.accessibility.AccessibilityEvent;
@@ -484,7 +486,6 @@ public class WebContentsAccessibilityTest {
     @MediumTest
     @MinAndroidSdkLevel(Build.VERSION_CODES.LOLLIPOP)
     @TargetApi(Build.VERSION_CODES.LOLLIPOP)
-
     public void testEditTextFieldValidNoErrorMessage() {
         final String data = "<input type='text'><br>\n";
         mActivityTestRule.launchContentShellWithUrl(UrlUtils.encodeHtmlDataUri(data));
@@ -498,4 +499,48 @@ public class WebContentsAccessibilityTest {
         Assert.assertEquals(textNode.isContentInvalid(), false);
         Assert.assertEquals(textNode.getError(), "");
     }
+
+    /**
+     * Test spelling error is encoded as a Spannable.
+     **/
+    @Test
+    @MediumTest
+    @MinAndroidSdkLevel(Build.VERSION_CODES.LOLLIPOP)
+    @TargetApi(Build.VERSION_CODES.LOLLIPOP)
+    public void testSpellingError() {
+        // Load a web page containing a text field with one misspelling.
+        // Note that for content_shell, no spelling suggestions are enabled
+        // by default.
+        final String data = "<input type='text' value='one wordd has an error'>";
+        mActivityTestRule.launchContentShellWithUrl(UrlUtils.encodeHtmlDataUri(data));
+        mActivityTestRule.waitForActiveShellToBeDoneLoading();
+        AccessibilityNodeProvider provider = enableAccessibilityAndWaitForNodeProvider();
+        int textNodeVirtualViewId = waitForNodeWithClassName(provider, "android.widget.EditText");
+
+        // Call a test API to explicitly add a spelling error in the same format as
+        // would be generated if spelling correction was enabled.
+        final WebContentsAccessibilityImpl wcax = mActivityTestRule.getWebContentsAccessibility();
+        wcax.addSpellingErrorForTesting(textNodeVirtualViewId, 4, 9);
+
+        // Now get that AccessibilityNodeInfo and retrieve its text.
+        AccessibilityNodeInfo textNode =
+                provider.createAccessibilityNodeInfo(textNodeVirtualViewId);
+        Assert.assertNotEquals(textNode, null);
+        CharSequence text = textNode.getText();
+        Assert.assertEquals(text.toString(), "one wordd has an error");
+
+        // Assert that the text has a SuggestionSpan surrounding the proper word.
+        Assert.assertTrue(text instanceof Spannable);
+        Spannable spannable = (Spannable) text;
+        Object spans[] = spannable.getSpans(0, text.length(), Object.class);
+        boolean foundSuggestionSpan = false;
+        for (Object span : spans) {
+            if (span instanceof SuggestionSpan) {
+                Assert.assertEquals(4, spannable.getSpanStart(span));
+                Assert.assertEquals(9, spannable.getSpanEnd(span));
+                foundSuggestionSpan = true;
+            }
+        }
+        Assert.assertTrue(foundSuggestionSpan);
+    }
 }