Force VoiceOver to announce selected state in multiselects

Send a localized string that includes the selection state to
the VoiceOver announcement API.
VoiceOver will speak the string and render on a Braille display.

Bug: 939526
Change-Id: I4aedd89bc4b7c99133afd9ca99eeddfe90f771c3
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1953286Reviewed-by: Nate Chapin <japhet@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Commit-Queue: Aaron Leventhal <aleventhal@chromium.org>
Cr-Commit-Position: refs/heads/master@{#723430}

content/browser/accessibility/browser_accessibility_cocoa.mm

@@ -690,6 +690,31 @@ void AppendTextToString(const std::string& extra_text, std::string* string) {
   *string += std::string(". ") + extra_text;
 }
 
+bool IsSelectedStateRelevant(BrowserAccessibility* item) {
+  if (!item->HasBoolAttribute(ax::mojom::BoolAttribute::kSelected))
+    return false;  // Does not have selected state -> not relevant.
+
+  BrowserAccessibility* container = item->PlatformGetSelectionContainer();
+  if (!container)
+    return false;  // No container -> not relevant.
+
+  if (container->HasState(ax::mojom::State::kMultiselectable))
+    return true;  // In a multiselectable -> is relevant.
+
+  // Single selection AND not selected - > is relevant.
+  // Single selection containers can explicitly set the focused item as not
+  // selected, for example via aria-selectable="false". It's useful for the user
+  // to know that it's not selected in this case.
+  // Only do this for the focused item -- that is the only item where explicitly
+  // setting the item to unselected is relevant, as the focused item is the only
+  // item that could have been selected annyway.
+  // Therefore, if the user navigates to other items by detaching accessibility
+  // focus from the input focus via VO+Shift+F3, those items will not be
+  // redundantly reported as not selected.
+  return item->manager()->GetFocus() == item &&
+         !item->GetBoolAttribute(ax::mojom::BoolAttribute::kSelected);
+}
+
 }  // namespace
 
 #if defined(MAC_OS_X_VERSION_10_12) && \
@@ -2345,9 +2370,25 @@ NSString* const NSAccessibilityRequiredAttributeChrome = @"AXRequired";
   if (![self instanceActive])
     return nil;
 
-  if (ui::IsNameExposedInAXValueForRole([self internalRole]))
-    return NSStringForStringAttribute(owner_,
-                                      ax::mojom::StringAttribute::kName);
+  if (ui::IsNameExposedInAXValueForRole([self internalRole])) {
+    if (!IsSelectedStateRelevant(owner_))
+      return NSStringForStringAttribute(owner_,
+                                        ax::mojom::StringAttribute::kName);
+
+    // Append the selection state as a string, because VoiceOver will not
+    // automatically report selection state when an individual item is focused.
+    base::string16 name =
+        owner_->GetString16Attribute(ax::mojom::StringAttribute::kName);
+    bool is_selected =
+        owner_->GetBoolAttribute(ax::mojom::BoolAttribute::kSelected);
+    int msg_id =
+        is_selected ? IDS_AX_OBJECT_SELECTED : IDS_AX_OBJECT_NOT_SELECTED;
+    ContentClient* content_client = content::GetContentClient();
+    base::string16 name_with_selection = base::ReplaceStringPlaceholders(
+        content_client->GetLocalizedString(msg_id), {name}, nullptr);
+
+    return base::SysUTF16ToNSString(name_with_selection);
+  }
 
   NSString* role = [self role];
   if ([role isEqualToString:@"AXHeading"]) {

content/browser/accessibility/browser_accessibility_mac.h

@@ -43,6 +43,10 @@ class BrowserAccessibilityMac : public BrowserAccessibility {
     return browser_accessibility_cocoa_;
   }
 
+  // Refresh the native object associated with this.
+  // Useful for re-announcing the current focus when properties have changed.
+  void ReplaceNativeObject();
+
  private:
   // This gives BrowserAccessibility::Create access to the class constructor.
   friend class BrowserAccessibility;

content/browser/accessibility/browser_accessibility_mac.mm

@@ -6,6 +6,8 @@
 
 #import "content/browser/accessibility/browser_accessibility_mac.h"
 
+#include "base/task/post_task.h"
+#include "base/time/time.h"
 #import "content/browser/accessibility/browser_accessibility_cocoa.h"
 #include "content/browser/accessibility/browser_accessibility_manager_mac.h"
 
@@ -47,6 +49,58 @@ void BrowserAccessibilityMac::OnDataChanged() {
       [[BrowserAccessibilityCocoa alloc] initWithObject:this];
 }
 
+// Replace a native object and refocus if it had focus.
+// This will force VoiceOver to re-announce it, and refresh Braille output.
+void BrowserAccessibilityMac::ReplaceNativeObject() {
+  BrowserAccessibilityCocoa* old_native_obj = browser_accessibility_cocoa_;
+  browser_accessibility_cocoa_ =
+      [[BrowserAccessibilityCocoa alloc] initWithObject:this];
+
+  // Replace child in parent.
+  BrowserAccessibility* parent = PlatformGetParent();
+  if (!parent)
+    return;
+
+  base::scoped_nsobject<NSMutableArray> new_children;
+  NSArray* old_children = [ToBrowserAccessibilityCocoa(parent) children];
+  for (uint i = 0; i < [old_children count]; ++i) {
+    BrowserAccessibilityCocoa* child = [old_children objectAtIndex:i];
+    if (child == old_native_obj)
+      [new_children addObject:browser_accessibility_cocoa_];
+    else
+      [new_children addObject:child];
+  }
+  [ToBrowserAccessibilityCocoa(parent) swapChildren:&new_children];
+
+  // If focused, fire a focus notification on the new native object.
+  if (manager_->GetFocus() == this) {
+    NSAccessibilityPostNotification(
+        browser_accessibility_cocoa_,
+        NSAccessibilityFocusedUIElementChangedNotification);
+  }
+
+  // Destroy after a delay so that VO is securely on the new focus first,
+  // otherwise the focus event will not be announced.
+  // We use 1000ms; however, this magic number isn't necessary to avoid
+  // use-after-free or anything scary like that. The worst case scenario if this
+  // gets destroyed, too early is that VoiceOver announces the wrong thing once.
+  base::scoped_nsobject<BrowserAccessibilityCocoa> retained_destroyed_node(
+      [old_native_obj retain]);
+
+  base::PostDelayedTask(
+      FROM_HERE,
+      base::BindOnce(
+          [](base::scoped_nsobject<BrowserAccessibilityCocoa> destroyed) {
+            if (destroyed && [destroyed instanceActive]) {
+              // Follow destruction pattern from NativeReleaseReference().
+              [destroyed detach];
+              [destroyed release];
+            }
+          },
+          std::move(retained_destroyed_node)),
+      base::TimeDelta::FromMilliseconds(1000));
+}
+
 uint32_t BrowserAccessibilityMac::PlatformChildCount() const {
   uint32_t child_count = BrowserAccessibility::PlatformChildCount();
 

content/browser/accessibility/browser_accessibility_manager_mac.h

@@ -66,6 +66,8 @@ class CONTENT_EXPORT BrowserAccessibilityManagerMac
 
   void AnnounceActiveDescendant(BrowserAccessibility* node) const;
 
+  bool IsInGeneratedEventBatch(ui::AXEventGenerator::Event event_type) const;
+
   // Keeps track of any edits that have been made by the user during a tree
   // update. Used by NSAccessibilityValueChangedNotification.
   // Maps AXNode IDs to value attribute changes.

content/browser/accessibility/browser_accessibility_manager_mac.mm

@@ -150,11 +150,6 @@ BrowserAccessibility* BrowserAccessibilityManagerMac::GetFocus() const {
   if (focus->GetRole() == ax::mojom::Role::kTextFieldWithComboBox)
     return focus;
 
-  // If multiselectable, treat the container as focused and send selected
-  // children changed events as the user navigates.
-  if (focus->HasState(ax::mojom::State::kMultiselectable))
-    return focus;
-
   // Otherwise, follow the active descendant.
   return GetActiveDescendant(focus);
 }
@@ -164,17 +159,6 @@ void BrowserAccessibilityManagerMac::FireFocusEvent(
   BrowserAccessibilityManager::FireFocusEvent(node);
   FireNativeMacNotification(NSAccessibilityFocusedUIElementChangedNotification,
                             node);
-
-  // Announce any active unselected item directly when a multiselection
-  // container becomes focused. VoiceOver will only automatically announce
-  // a selected item, and the active descendant may not be selected.
-  if (node && node->HasState(ax::mojom::State::kMultiselectable)) {
-    BrowserAccessibility* active_descendant = GetActiveDescendant(node);
-    if (active_descendant->GetBoolAttribute(
-            ax::mojom::BoolAttribute::kSelected))
-      return;  // Selected item will already be announced upon new focus.
-    AnnounceActiveDescendant(node);
-  }
 }
 
 void BrowserAccessibilityManagerMac::FireBlinkEvent(
@@ -201,21 +185,22 @@ void PostAnnouncementNotification(NSString* announcement) {
     NSAccessibilityAnnouncementKey : announcement,
     NSAccessibilityPriorityKey : @(NSAccessibilityPriorityLow)
   };
+  // Trigger VoiceOver speech and show on Braille display, if available.
+  // The Braille will only appear for a few seconds, and then will be replaced
+  // with the previous announcement.
   NSAccessibilityPostNotificationWithUserInfo(
       [NSApp mainWindow], NSAccessibilityAnnouncementRequestedNotification,
       notification_info);
 }
 
-void BrowserAccessibilityManagerMac::AnnounceActiveDescendant(
-    BrowserAccessibility* node) const {
-  if (GetFocus() != node)
-    return;  // Container not focused.
-  BrowserAccessibility* active_descendant = GetActiveDescendant(node);
-  if (!active_descendant || active_descendant == node)
-    return;  // No active descendant.
-  PostAnnouncementNotification(
-      base::SysUTF8ToNSString(active_descendant->GetStringAttribute(
-          ax::mojom::StringAttribute::kName)));
+// Check whether the current batch of events contains the event type.
+bool BrowserAccessibilityManagerMac::IsInGeneratedEventBatch(
+    ui::AXEventGenerator::Event event_type) const {
+  for (const auto& event : event_generator_) {
+    if (event.event_params.event == event_type)
+      return true;  // Announcement will already be handled via this event.
+  }
+  return false;
 }
 
 void BrowserAccessibilityManagerMac::FireGeneratedEvent(
@@ -239,12 +224,6 @@ void BrowserAccessibilityManagerMac::FireGeneratedEvent(
         // want to post a focus change because this will take the focus out of
         // the combo box where the user might be typing.
         mac_notification = NSAccessibilitySelectedChildrenChangedNotification;
-      } else if (node->HasState(ax::mojom::State::kMultiselectable)) {
-        // Speak item directly in case the focused item changes but selection
-        // does not change, e.g. if cmd+down is pressed in a multiselectable
-        // listbox, so that there is still some announcement.
-        AnnounceActiveDescendant(node);
-        return;
       } else {
         // In all other cases we should post
         // |NSAccessibilityFocusedUIElementChangedNotification|, but this is
@@ -269,6 +248,30 @@ void BrowserAccessibilityManagerMac::FireGeneratedEvent(
       if (ui::IsTableLike(node->GetRole())) {
         mac_notification = NSAccessibilitySelectedRowsChangedNotification;
       } else {
+        // VoiceOver does not read anything if selection changes on the
+        // currently focused object, and the focus did not move. Detect a
+        // selection change in a where the focus did not change.
+        BrowserAccessibility* focus = GetFocus();
+        BrowserAccessibility* container =
+            focus->PlatformGetSelectionContainer();
+
+        if (focus && node == container &&
+            container->HasState(ax::mojom::State::kMultiselectable) &&
+            !IsInGeneratedEventBatch(
+                ui::AXEventGenerator::Event::ACTIVE_DESCENDANT_CHANGED) &&
+            !IsInGeneratedEventBatch(
+                ui::AXEventGenerator::Event::FOCUS_CHANGED)) {
+          // Force announcement of current focus / activedescendant, even though
+          // it's not changing. This way, the user can hear the new selection
+          // state of the current object. Because VoiceOver ignores focus events
+          // to an already focused object, this is done by destroying the native
+          // object and creating a new one that receives focus.
+          static_cast<BrowserAccessibilityMac*>(focus)->ReplaceNativeObject();
+          // Don't fire selected children change, it will sometimes override
+          // announcement of current focus.
+          return;
+        }
+
         mac_notification = NSAccessibilitySelectedChildrenChangedNotification;
       }
       break;

content/test/data/accessibility/aria/aria-combobox-expected-mac.txt

@@ -4,4 +4,4 @@ AXWebArea
 ++AXComboBox AXTitle='State' AXAutocompleteValue='list' AXFocused='1'
 ++AXList
 ++++AXStaticText AXValue='Alabama'
-++++AXStaticText AXValue='Alaska'
+++++AXStaticText AXValue='Alaska'
\ No newline at end of file

content/test/data/accessibility/aria/aria-listbox-aria-selected-expected-mac.txt

 AXWebArea AXSelected='0'
-++AXList AXOrientation='AXVerticalOrientation' AXSelected='0' AXSelectedChildren=["AXStaticText Item 4","AXStaticText Item 5"] AXVisibleChildren=["AXStaticText Item 1","AXStaticText Item 2","AXStaticText Item 3","AXStaticText Item 4","AXStaticText Item 5"]
-++++AXStaticText AXValue='Item 1' AXSelected='0'
-++++AXStaticText AXValue='Item 2' AXSelected='0'
-++++AXStaticText AXValue='Item 3' AXSelected='0'
-++++AXStaticText AXValue='Item 4' AXSelected='1'
-++++AXStaticText AXValue='Item 5' AXSelected='1'
+++AXList AXOrientation='AXVerticalOrientation' AXSelected='0' AXSelectedChildren=["AXStaticText Item 4 selected","AXStaticText Item 5 selected"] AXVisibleChildren=["AXStaticText Item 1 not selected","AXStaticText Item 2 not selected","AXStaticText Item 3 not selected","AXStaticText Item 4 selected","AXStaticText Item 5 selected"]
+++++AXStaticText AXValue='Item 1 not selected' AXSelected='0'
+++++AXStaticText AXValue='Item 2 not selected' AXSelected='0'
+++++AXStaticText AXValue='Item 3 not selected' AXSelected='0'
+++++AXStaticText AXValue='Item 4 selected' AXSelected='1'
+++++AXStaticText AXValue='Item 5 selected' AXSelected='1'
\ No newline at end of file

content/test/data/accessibility/aria/aria-listbox-childfocus-expected-mac.txt

@@ -2,4 +2,4 @@ AXWebArea
 ++AXList AXOrientation='AXVerticalOrientation' AXSelectedChildren=["AXStaticText 2"] AXVisibleChildren=["AXStaticText 1","AXStaticText 2","AXStaticText 3"]
 ++++AXStaticText AXValue='1'
 ++++AXStaticText AXValue='2'
-++++AXStaticText AXValue='3'
+++++AXStaticText AXValue='3'
\ No newline at end of file

content/test/data/accessibility/aria/aria-option-expected-mac.txt

 AXWebArea AXRoleDescription='HTML content'
 ++AXList AXRoleDescription='list'
 ++++AXStaticText AXRoleDescription='text' AXValue='option 1'
-++++AXStaticText AXRoleDescription='text' AXValue='label 2'
+++++AXStaticText AXRoleDescription='text' AXValue='label 2'
\ No newline at end of file

content/test/data/accessibility/aria/aria-posinset-expected-mac.txt

@@ -33,4 +33,4 @@ AXWebArea AXRoleDescription='HTML content'
 ++++++++AXStaticText AXRoleDescription='text' AXValue='red'
 ++++++AXGroup AXRoleDescription='group' AXTitle='<newline>'
 ++++++AXRadioButton AXRoleDescription='radio button' AXTitle='blue' AXValue='0' AXARIASetSize='1' AXARIAPosInSet='1'
-++AXStaticText AXRoleDescription='text' AXValue='Done'
+++AXStaticText AXRoleDescription='text' AXValue='Done'
\ No newline at end of file

content/test/data/accessibility/aria/aria-setsize-expected-mac.txt

@@ -9,4 +9,4 @@ AXWebArea AXRoleDescription='HTML content'
 ++++AXStaticText AXRoleDescription='text' AXValue='Item 2' AXARIASetSize='5' AXARIAPosInSet='2'
 ++++AXStaticText AXRoleDescription='text' AXValue='Item 3' AXARIASetSize='5' AXARIAPosInSet='3'
 ++++AXStaticText AXRoleDescription='text' AXValue='Item 4' AXARIASetSize='5' AXARIAPosInSet='4'
-++++AXStaticText AXRoleDescription='text' AXValue='Item 5' AXARIASetSize='5' AXARIAPosInSet='5'
+++++AXStaticText AXRoleDescription='text' AXValue='Item 5' AXARIASetSize='5' AXARIAPosInSet='5'
\ No newline at end of file

content/test/data/accessibility/aria/aria1.1-combobox-expected-mac.txt

@@ -10,4 +10,4 @@ AXWebArea
 ++++AXTextField AXLinkedUIElements=["AXList"]
 ++AXList
 ++++AXStaticText AXValue='Alabama'
-++++AXStaticText AXValue='Alaska'
+++++AXStaticText AXValue='Alaska'
\ No newline at end of file

content/test/data/accessibility/event/focus-listbox-multiselect-expected-mac.txt

-AXFocusedUIElementChanged on AXList
-AXSelectedChildrenChanged on AXList
+AXFocusedUIElementChanged on AXStaticText AXValue="c selected"
+AXSelectedChildrenChanged on AXList
\ No newline at end of file

content/test/data/accessibility/html/button-with-listbox-popup-expected-mac.txt

@@ -7,4 +7,4 @@ AXWebArea
 ++++AXList AXTitle='Choose one:'
 ++++++AXStaticText AXValue='Baz'
 ++++++AXStaticText AXValue='Bar'
-++++++AXStaticText AXValue='Foo'
+++++++AXStaticText AXValue='Foo'
\ No newline at end of file

content/test/data/accessibility/html/optgroup-expected-mac.txt

@@ -12,4 +12,4 @@ AXWebArea AXRoleDescription='HTML content'
 ++++++AXStaticText AXRoleDescription='text' AXValue='One'
 ++++++AXStaticText AXRoleDescription='text' AXValue='Two'
 ++++++AXStaticText AXRoleDescription='text' AXValue='Three'
-++++++AXStaticText AXRoleDescription='text' AXValue='Four'
+++++++AXStaticText AXRoleDescription='text' AXValue='Four'
\ No newline at end of file

content/test/data/accessibility/html/select-expected-mac.txt

@@ -16,10 +16,10 @@ AXWebArea AXRoleDescription='HTML content'
 ++++++++AXMenuItem AXRoleDescription='menu item' AXValue='Option 2'
 ++++++++AXMenuItem AXRoleDescription='menu item' AXValue='Option 3'
 ++++AXList AXRoleDescription='list'
-++++++AXStaticText AXRoleDescription='text' AXValue='Option 1'
-++++++AXStaticText AXRoleDescription='text' AXValue='Option 2'
-++++++AXStaticText AXRoleDescription='text' AXValue='Option 3'
+++++++AXStaticText AXRoleDescription='text' AXValue='Option 1 not selected'
+++++++AXStaticText AXRoleDescription='text' AXValue='Option 2 not selected'
+++++++AXStaticText AXRoleDescription='text' AXValue='Option 3 not selected'
 ++++AXList AXRoleDescription='list'
 ++++++AXStaticText AXRoleDescription='text' AXValue='Option 1'
 ++++++AXStaticText AXRoleDescription='text' AXValue='Option 2'
-++++++AXStaticText AXRoleDescription='text' AXValue='Option 3'
+++++++AXStaticText AXRoleDescription='text' AXValue='Option 3'
\ No newline at end of file

third_party/blink/public/strings/blink_strings.grd

@@ -889,6 +889,14 @@ below:
         Year
       </message>
 
+      <message name="IDS_AX_OBJECT_SELECTED" desc="The current object is selected, like an option in a list">
+        <ph name="accName">$1<ex>Hazelnuts</ex></ph> selected
+      </message>
+      <message name="IDS_AX_OBJECT_NOT_SELECTED" desc="The current object is selected, like an option in a list">
+        <ph name="accName">$1<ex>Hazelnuts</ex></ph> not selected
+      </message>
+
+
       <message name="IDS_FORM_INPUT_WEEK_TEMPLATE" desc="A specific week (1-53) in a specific year shown in a form control">
         Week <ph name="WEEKNUMBER">$2<ex>51</ex></ph>, <ph name="YEAR">$1<ex>2012</ex></ph>
       </message>