Add accessibility support for multiselectable aria-activedescendants on Android

This CL adds support for multiselectable aria-activedescendants

For nodes that are multiselectable (e.g. listbox of items with an
aria-activedescendant role), we add more descriptive labels. For
Android R we use the new stateDescription on AccessibilityNodeInfo
object for the given node. To support pre-Android R, we append the
labels to the text for the node.

For a multiselectable node that has no items selected, we add the
text:

"multiselectable, none selected."

For a multiselectable node with n children selected of k total, we
add the text:

"multiselectable, n of k selected."



multiselectable aria activedescendants

AX-Relnotes: Added more descriptive accessibility labels for
Change-Id: I9fe04714c5800939b73b11c3339e628ae7760939
Bug: 1101660
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2341826Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: Scott Violet <sky@chromium.org>
Commit-Queue: Mark Schillaci <mschillaci@google.com>
Cr-Commit-Position: refs/heads/master@{#798926}

content/browser/accessibility/accessibility_tree_formatter_android.cc

@@ -22,24 +22,39 @@ using base::StringPrintf;
 namespace content {
 
 namespace {
-
+// clang-format off
 const char* const BOOL_ATTRIBUTES[] = {
-    "checkable",       "checked",
-    "clickable",       "collection",
-    "collection_item", "content_invalid",
-    "disabled",        "dismissable",
-    "editable_text",   "focusable",
-    "focused",         "has_character_locations",
-    "has_image",       "has_non_empty_value",
-    "heading",         "hierarchical",
-    "invisible",       "link",
-    "multiline",       "password",
-    "range",           "scrollable",
-    "selected",        "interesting"};
+    "checkable",
+    "checked",
+    "clickable",
+    "collection",
+    "collection_item",
+    "content_invalid",
+    "disabled",
+    "dismissable",
+    "editable_text",
+    "focusable",
+    "focused",
+    "has_character_locations",
+    "has_image",
+    "has_non_empty_value",
+    "heading",
+    "hierarchical",
+    "invisible",
+    "link",
+    "multiline",
+    "multiselectable",
+    "password",
+    "range",
+    "scrollable",
+    "selected",
+    "interesting"
+};
 
 const char* const STRING_ATTRIBUTES[] = {
     "name",
     "hint",
+    "state_description",
 };
 
 const char* const INT_ATTRIBUTES[] = {
@@ -59,7 +74,7 @@ const char* const INT_ATTRIBUTES[] = {
     "text_change_added_count",
     "text_change_removed_count",
 };
-
+// clang-format on
 }  // namespace
 
 class AccessibilityTreeFormatterAndroid
@@ -211,6 +226,7 @@ void AccessibilityTreeFormatterAndroid::AddProperties(
   dict->SetBoolean("invisible", !android_node->IsVisibleToUser());
   dict->SetBoolean("link", android_node->IsLink());
   dict->SetBoolean("multiline", android_node->IsMultiLine());
+  dict->SetBoolean("multiselectable", android_node->IsMultiselectable());
   dict->SetBoolean("range", android_node->IsRangeType());
   dict->SetBoolean("password", android_node->IsPasswordField());
   dict->SetBoolean("scrollable", android_node->IsScrollable());
@@ -221,6 +237,7 @@ void AccessibilityTreeFormatterAndroid::AddProperties(
   dict->SetString("name", android_node->GetInnerText());
   dict->SetString("hint", android_node->GetHint());
   dict->SetString("role_description", android_node->GetRoleDescription());
+  dict->SetString("state_description", android_node->GetStateDescription());
 
   // Int attributes.
   dict->SetInteger("item_index", android_node->GetItemIndex());

content/browser/accessibility/browser_accessibility_android.cc

@@ -253,6 +253,10 @@ bool BrowserAccessibilityAndroid::IsMultiLine() const {
   return HasState(ax::mojom::State::kMultiline);
 }
 
+bool BrowserAccessibilityAndroid::IsMultiselectable() const {
+  return HasState(ax::mojom::State::kMultiselectable);
+}
+
 bool BrowserAccessibilityAndroid::IsRangeType() const {
   return (GetRole() == ax::mojom::Role::kProgressIndicator ||
           GetRole() == ax::mojom::Role::kMeter ||
@@ -534,6 +538,47 @@ base::string16 BrowserAccessibilityAndroid::GetHint() const {
   return base::JoinString(strings, base::ASCIIToUTF16(" "));
 }
 
+base::string16 BrowserAccessibilityAndroid::GetStateDescription() const {
+  // For multiselectable state, generate a state description
+  if (IsMultiselectable())
+    return GetMultiselectableStateDescription();
+
+  // Otherwise we will not use state description
+  return base::string16();
+}
+
+base::string16 BrowserAccessibilityAndroid::GetMultiselectableStateDescription()
+    const {
+  content::ContentClient* content_client = content::GetContentClient();
+
+  // Count the number of children and selected children.
+  int child_count = 0;
+  int selected_count = 0;
+  for (PlatformChildIterator it = PlatformChildrenBegin();
+       it != PlatformChildrenEnd(); ++it) {
+    child_count++;
+    BrowserAccessibilityAndroid* child =
+        static_cast<BrowserAccessibilityAndroid*>(it.get());
+    if (child->IsSelected())
+      selected_count++;
+  }
+
+  // If none are selected, return special case.
+  if (!selected_count)
+    return content_client->GetLocalizedString(
+        IDS_AX_MULTISELECTABLE_STATE_DESCRIPTION_NONE);
+
+  // Generate a state description of the form: "multiselectable, x of y
+  // selected.".
+  std::vector<base::string16> values;
+  values.push_back(base::NumberToString16(selected_count));
+  values.push_back(base::NumberToString16(child_count));
+  return base::ReplaceStringPlaceholders(
+      content_client->GetLocalizedString(
+          IDS_AX_MULTISELECTABLE_STATE_DESCRIPTION),
+      values, nullptr);
+}
+
 std::string BrowserAccessibilityAndroid::GetRoleString() const {
   return ui::ToString(GetRole());
 }

content/browser/accessibility/browser_accessibility_android.h

@@ -45,6 +45,7 @@ class CONTENT_EXPORT BrowserAccessibilityAndroid : public BrowserAccessibility {
   bool IsHierarchical() const;
   bool IsLink() const;
   bool IsMultiLine() const;
+  bool IsMultiselectable() const;
   bool IsRangeType() const;
   bool IsScrollable() const;
   bool IsSelected() const;
@@ -88,6 +89,9 @@ class CONTENT_EXPORT BrowserAccessibilityAndroid : public BrowserAccessibility {
 
   base::string16 GetContentInvalidErrorMessage() const;
 
+  base::string16 GetStateDescription() const;
+  base::string16 GetMultiselectableStateDescription() const;
+
   base::string16 GetRoleDescription() const;
 
   int GetItemIndex() const;

content/browser/accessibility/web_contents_accessibility_android.cc

@@ -797,7 +797,9 @@ jboolean WebContentsAccessibilityAndroid::PopulateAccessibilityNodeInfo(
       base::android::ConvertUTF16ToJavaString(
           env, node->GetInheritedString16Attribute(
                    ax::mojom::StringAttribute::kLanguage)),
-      suggestion_starts_java, suggestion_ends_java, suggestion_text_java);
+      suggestion_starts_java, suggestion_ends_java, suggestion_text_java,
+      base::android::ConvertUTF16ToJavaString(env,
+                                              node->GetStateDescription()));
 
   base::string16 element_id;
   if (node->GetHtmlAttribute("id", &element_id)) {

content/public/android/BUILD.gn

@@ -187,6 +187,7 @@ android_library("content_java") {
     "java/src/org/chromium/content/browser/accessibility/BrowserAccessibilityState.java",
     "java/src/org/chromium/content/browser/accessibility/OWebContentsAccessibility.java",
     "java/src/org/chromium/content/browser/accessibility/PieWebContentsAccessibility.java",
+    "java/src/org/chromium/content/browser/accessibility/RWebContentsAccessibility.java",
     "java/src/org/chromium/content/browser/accessibility/WebContentsAccessibilityImpl.java",
     "java/src/org/chromium/content/browser/accessibility/captioning/CaptioningBridge.java",
     "java/src/org/chromium/content/browser/accessibility/captioning/CaptioningChangeDelegate.java",

content/public/android/java/src/org/chromium/content/browser/accessibility/RWebContentsAccessibility.java

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package org.chromium.content.browser.accessibility;
+
+import android.annotation.TargetApi;
+import android.os.Build;
+import android.view.accessibility.AccessibilityNodeInfo;
+
+import org.chromium.base.annotations.JNINamespace;
+import org.chromium.content_public.browser.WebContents;
+
+/**
+ * Subclass of WebContentsAccessibility for R
+ */
+@JNINamespace("content")
+@TargetApi(Build.VERSION_CODES.R)
+public class RWebContentsAccessibility extends PieWebContentsAccessibility {
+    RWebContentsAccessibility(WebContents webContents) {
+        super(webContents);
+    }
+
+    @Override
+    protected void setAccessibilityNodeInfoText(AccessibilityNodeInfo node, String text,
+            boolean annotateAsLink, boolean isEditableText, String language, int[] suggestionStarts,
+            int[] suggestionEnds, String[] suggestions, String stateDescription) {
+        super.setAccessibilityNodeInfoText(node, text, annotateAsLink, isEditableText, language,
+                suggestionStarts, suggestionEnds, suggestions, stateDescription);
+
+        // For Android R and higher, we will not rely on concatenating text and stateDescription,
+        // and will instead revert text to original content and set stateDescription separately.
+        if (stateDescription != null && !stateDescription.isEmpty()) {
+            CharSequence computedText = computeText(
+                    text, isEditableText, language, suggestionStarts, suggestionEnds, suggestions);
+
+            node.setText(computedText);
+            node.setStateDescription(stateDescription);
+        }
+    }
+}

content/public/android/java/src/org/chromium/content/browser/accessibility/WebContentsAccessibilityImpl.java

@@ -203,7 +203,9 @@ public class WebContentsAccessibilityImpl extends AccessibilityNodeProvider
     private static class Factory implements UserDataFactory<WebContentsAccessibilityImpl> {
         @Override
         public WebContentsAccessibilityImpl create(WebContents webContents) {
-            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+                return new RWebContentsAccessibility(webContents);
+            } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
                 return new PieWebContentsAccessibility(webContents);
             } else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
                 return new OWebContentsAccessibility(webContents);
@@ -1552,9 +1554,9 @@ public class WebContentsAccessibilityImpl extends AccessibilityNodeProvider
 
     @SuppressLint("NewApi")
     @CalledByNative
-    private void setAccessibilityNodeInfoText(AccessibilityNodeInfo node, String text,
+    protected void setAccessibilityNodeInfoText(AccessibilityNodeInfo node, String text,
             boolean annotateAsLink, boolean isEditableText, String language, int[] suggestionStarts,
-            int[] suggestionEnds, String[] suggestions) {
+            int[] suggestionEnds, String[] suggestions, String stateDescription) {
         CharSequence computedText = computeText(
                 text, isEditableText, language, suggestionStarts, suggestionEnds, suggestions);
         // We expose the nested structure of links, which results in the roles of all nested nodes
@@ -1564,6 +1566,11 @@ public class WebContentsAccessibilityImpl extends AccessibilityNodeProvider
         } else {
             node.setText(computedText);
         }
+
+        // For pre-Android R, we add stateDescription to text for backwards compatibility.
+        if (stateDescription != null && !stateDescription.isEmpty()) {
+            node.setText(computedText + ", " + stateDescription);
+        }
     }
 
     protected CharSequence computeText(String text, boolean annotateAsLink, String language,

content/test/data/accessibility/aria/aria-multiselectable-expected-android.txt

 android.webkit.WebView focusable focused scrollable
-++android.widget.ListView role_description='list box' clickable collection
-++android.widget.ListView role_description='list box' clickable collection
\ No newline at end of file
+++android.widget.ListView role_description='list box' clickable collection focusable multiselectable name='My Listbox' state_description='multiselectable, none selected.' item_count=4 row_count=4
+++++android.view.View clickable collection_item focusable name='Example 1'
+++++android.view.View clickable collection_item focusable name='Example 2' item_index=1 row_index=1
+++++android.view.View clickable collection_item focusable name='Example 3' item_index=2 row_index=2
+++++android.view.View clickable collection_item focusable name='Example 4' item_index=3 row_index=3
+++android.widget.ListView role_description='list box' clickable collection focusable multiselectable name='My Listbox' state_description='multiselectable, 2 of 4 selected.' item_count=4 row_count=4
+++++android.view.View clickable collection_item focusable name='Example 1'
+++++android.view.View clickable collection_item focusable selected name='Example 2' item_index=1 row_index=1
+++++android.view.View clickable collection_item focusable name='Example 3' item_index=2 row_index=2
+++++android.view.View clickable collection_item focusable selected name='Example 4' item_index=3 row_index=3
\ No newline at end of file

content/test/data/accessibility/aria/aria-multiselectable-expected-auralinux.txt

 [document web]
-++[list box] multiselectable
-++[list box]
+++[list box] name='My Listbox' multiselectable
+++++[list item] name='Example 1' selectable
+++++[list item] name='Example 2' selectable
+++++[list item] name='Example 3' selectable
+++++[list item] name='Example 4' selectable
+++[list box] name='My Listbox' multiselectable
+++++[list item] name='Example 1' selectable
+++++[list item] name='Example 2' selectable selected
+++++[list item] name='Example 3' selectable
+++++[list item] name='Example 4' selectable selected
\ No newline at end of file

content/test/data/accessibility/aria/aria-multiselectable-expected-blink.txt

 rootWebArea
 ++genericContainer ignored
 ++++genericContainer ignored
-++++++listBox multiselectable
-++++++listBox
+++++++listBox multiselectable name='My Listbox'
+++++++++listBoxOption name='Example 1' selected=false
+++++++++listBoxOption name='Example 2' selected=false
+++++++++listBoxOption name='Example 3' selected=false
+++++++++listBoxOption name='Example 4' selected=false
+++++++listBox multiselectable name='My Listbox'
+++++++++listBoxOption name='Example 1' selected=false
+++++++++listBoxOption name='Example 2' selected=true
+++++++++listBoxOption name='Example 3' selected=false
+++++++++listBoxOption name='Example 4' selected=true
\ No newline at end of file

content/test/data/accessibility/aria/aria-multiselectable-expected-mac.txt

 AXWebArea
-++AXList
-++AXList
+++AXList AXDescription='My Listbox'
+++++AXStaticText AXValue='Example 1 not selected'
+++++AXStaticText AXValue='Example 2 not selected'
+++++AXStaticText AXValue='Example 3 not selected'
+++++AXStaticText AXValue='Example 4 not selected'
+++AXList AXDescription='My Listbox'
+++++AXStaticText AXValue='Example 1 not selected'
+++++AXStaticText AXValue='Example 2 selected'
+++++AXStaticText AXValue='Example 3 not selected'
+++++AXStaticText AXValue='Example 4 selected'
\ No newline at end of file

content/test/data/accessibility/aria/aria-multiselectable-expected-win.txt

 ROLE_SYSTEM_DOCUMENT READONLY FOCUSABLE
-++ROLE_SYSTEM_LIST MULTISELECTABLE EXTSELECTABLE xml-roles:listbox
-++ROLE_SYSTEM_LIST xml-roles:listbox
\ No newline at end of file
+++ROLE_SYSTEM_LIST name='My Listbox' FOCUSABLE MULTISELECTABLE EXTSELECTABLE xml-roles:listbox
+++++ROLE_SYSTEM_LISTITEM name='Example 1' FOCUSABLE xml-roles:option
+++++ROLE_SYSTEM_LISTITEM name='Example 2' FOCUSABLE xml-roles:option
+++++ROLE_SYSTEM_LISTITEM name='Example 3' FOCUSABLE xml-roles:option
+++++ROLE_SYSTEM_LISTITEM name='Example 4' FOCUSABLE xml-roles:option
+++ROLE_SYSTEM_LIST name='My Listbox' FOCUSABLE MULTISELECTABLE EXTSELECTABLE xml-roles:listbox
+++++ROLE_SYSTEM_LISTITEM name='Example 1' FOCUSABLE xml-roles:option
+++++ROLE_SYSTEM_LISTITEM name='Example 2' SELECTED FOCUSABLE xml-roles:option
+++++ROLE_SYSTEM_LISTITEM name='Example 3' FOCUSABLE xml-roles:option
+++++ROLE_SYSTEM_LISTITEM name='Example 4' SELECTED FOCUSABLE xml-roles:option
\ No newline at end of file

content/test/data/accessibility/aria/aria-multiselectable.html

@@ -2,11 +2,22 @@
 @WIN-ALLOW:xml-roles:*
 @WIN-ALLOW:MULTISELECTABLE
 @WIN-ALLOW:EXTSELECTABLE
+@ANDROID-ALLOW:state_description
 -->
 <!DOCTYPE html>
 <html>
 <body>
-  <div role="listbox" aria-multiselectable="true"></div>
-  <div role="listbox" aria-multiselectable="false"></div>
+  <ul tabindex="0" role="listbox" aria-multiselectable="true" aria-label="My Listbox">
+    <li role="option" aria-selected="false">Example 1</li>
+    <li role="option" aria-selected="false">Example 2</li>
+    <li role="option" aria-selected="false">Example 3</li>
+    <li role="option" aria-selected="false">Example 4</li>
+  </ul>
+  <ul tabindex="0" role="listbox" aria-multiselectable="true" aria-label="My Listbox">
+    <li role="option" aria-selected="false">Example 1</li>
+    <li role="option" aria-selected="true">Example 2</li>
+    <li role="option" aria-selected="false">Example 3</li>
+    <li role="option" aria-selected="true">Example 4</li>
+  </ul>
 </body>
 </html>

third_party/blink/public/strings/blink_strings.grd

@@ -778,6 +778,12 @@ below:
         <message name="IDS_AX_ROLE_TREE_ITEM" desc="Accessibility role description for a tree item">
           tree item
         </message>
+        <message name="IDS_AX_MULTISELECTABLE_STATE_DESCRIPTION" desc="Accessibility state description for a multiselectable node when some children are selected">
+          multiselectable, <ph name="SELECTED">$1<ex>3</ex></ph> of <ph name="COUNT">$2<ex>10</ex></ph> selected.
+        </message>
+        <message name="IDS_AX_MULTISELECTABLE_STATE_DESCRIPTION_NONE" desc="Accessibility state description for a multiselectable node when no children are selected">
+          multiselectable, none selected.
+        </message>
       </if>
 
       <!-- Automatic image annotations for accessibility -->

third_party/blink/public/strings/blink_strings_grd/IDS_AX_MULTISELECTABLE_STATE_DESCRIPTION.png.sha1

+7aea0db8d50cfd1db94a7c44fd56cb793dc98cb6
\ No newline at end of file

third_party/blink/public/strings/blink_strings_grd/IDS_AX_MULTISELECTABLE_STATE_DESCRIPTION_NONE.png.sha1

+3e059c4858bc962428a3ec4dc857f75d90f39c14
\ No newline at end of file