Attaches intents to accessibility text changed events

When an accessibility text or value changed event is raise, assistive software, such as
VoiceOver and ChromeVox, doesn't really know how the previous text was modified.
The event doesn't specify for example if:
a character has been typed, a new line inserted,
a piece of text has been cut, something was pasted, or
the current selection replaced with a new piece of text.
Assistive software, such as screen readers, need to either monitor key presses, or rely on heuristics to determine what to announce.

This patch adds such information to text and value changed events
through a list of event intents which describe the command that was carried out.

R=dmazzoni@chromium.org, yosin@chromium.org

AX-Relnotes: n/a
Bug: 989156
Change-Id: Ie33cfd73d195086c80883315ee8fef13bdff251e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2214951
Commit-Queue: Nektarios Paisios <nektar@chromium.org>
Auto-Submit: Nektarios Paisios <nektar@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: Yoshifumi Inoue <yosin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#803823}

third_party/blink/renderer/core/accessibility/blink_ax_event_intent.cc

@@ -6,11 +6,135 @@
 
 #include <limits>
 
+#include "third_party/blink/renderer/core/events/input_event.h"
 #include "third_party/blink/renderer/platform/wtf/hash_functions.h"
 #include "ui/accessibility/ax_enums.mojom-blink.h"
 
 namespace blink {
 
+// static
+BlinkAXEventIntent BlinkAXEventIntent::FromEditCommand(
+    const EditCommand& edit_command) {
+  ax::mojom::blink::Command command;
+  // Set default values for move direction and text boundary.
+  ax::mojom::blink::TextBoundary text_boundary =
+      ax::mojom::blink::TextBoundary::kCharacter;
+  ax::mojom::blink::MoveDirection move_direction =
+      ax::mojom::blink::MoveDirection::kForward;
+
+  switch (edit_command.GetInputType()) {
+    case InputEvent::InputType::kNone:
+      return BlinkAXEventIntent();  // An empty intent.
+
+    // Insertion.
+    case InputEvent::InputType::kInsertText:
+      command = ax::mojom::blink::Command::kType;
+      break;
+    case InputEvent::InputType::kInsertLineBreak:
+      command = ax::mojom::blink::Command::kType;
+      text_boundary = ax::mojom::blink::TextBoundary::kLineEnd;
+      break;
+    case InputEvent::InputType::kInsertParagraph:
+      command = ax::mojom::blink::Command::kType;
+      text_boundary = ax::mojom::blink::TextBoundary::kParagraphEnd;
+      break;
+    case InputEvent::InputType::kInsertOrderedList:
+    case InputEvent::InputType::kInsertUnorderedList:
+      command = ax::mojom::blink::Command::kFormat;
+      break;
+    case InputEvent::InputType::kInsertHorizontalRule:
+      command = ax::mojom::blink::Command::kType;
+      break;
+    case InputEvent::InputType::kInsertFromPaste:
+    case InputEvent::InputType::kInsertFromDrop:
+    case InputEvent::InputType::kInsertFromYank:
+      command = ax::mojom::blink::Command::kPaste;
+      break;
+    case InputEvent::InputType::kInsertTranspose:
+    case InputEvent::InputType::kInsertReplacementText:
+      command = ax::mojom::blink::Command::kReplace;
+      break;
+    case InputEvent::InputType::kInsertCompositionText:
+      command = ax::mojom::blink::Command::kType;
+      break;
+
+    // Deletion.
+    //
+    // Text boundary indicates up to which point the deletion is applied. For
+    // example, if a soft line break is deleted in the forward direction, then
+    // it means that we are deleting until the next line start.
+    case InputEvent::InputType::kDeleteWordBackward:
+      command = ax::mojom::blink::Command::kDelete;
+      text_boundary = ax::mojom::blink::TextBoundary::kWordStart;
+      move_direction = ax::mojom::blink::MoveDirection::kBackward;
+      break;
+    case InputEvent::InputType::kDeleteWordForward:
+      command = ax::mojom::blink::Command::kDelete;
+      text_boundary = ax::mojom::blink::TextBoundary::kWordEnd;
+      break;
+    case InputEvent::InputType::kDeleteSoftLineBackward:
+      command = ax::mojom::blink::Command::kDelete;
+      text_boundary = ax::mojom::blink::TextBoundary::kLineEnd;
+      move_direction = ax::mojom::blink::MoveDirection::kBackward;
+      break;
+    case InputEvent::InputType::kDeleteSoftLineForward:
+      command = ax::mojom::blink::Command::kDelete;
+      text_boundary = ax::mojom::blink::TextBoundary::kLineStart;
+      break;
+    case InputEvent::InputType::kDeleteHardLineBackward:
+      command = ax::mojom::blink::Command::kDelete;
+      text_boundary = ax::mojom::blink::TextBoundary::kParagraphEnd;
+      move_direction = ax::mojom::blink::MoveDirection::kBackward;
+      break;
+    case InputEvent::InputType::kDeleteHardLineForward:
+      command = ax::mojom::blink::Command::kDelete;
+      text_boundary = ax::mojom::blink::TextBoundary::kParagraphStart;
+      break;
+    case InputEvent::InputType::kDeleteContentBackward:
+      command = ax::mojom::blink::Command::kDelete;
+      move_direction = ax::mojom::blink::MoveDirection::kBackward;
+      break;
+    case InputEvent::InputType::kDeleteContentForward:
+      command = ax::mojom::blink::Command::kDelete;
+      break;
+    case InputEvent::InputType::kDeleteByCut:
+    case InputEvent::InputType::kDeleteByDrag:
+      command = ax::mojom::blink::Command::kCut;
+      break;
+
+    // History.
+    case InputEvent::InputType::kHistoryUndo:
+    case InputEvent::InputType::kHistoryRedo:
+      return BlinkAXEventIntent();  // No accessibility sideeffects for now.
+
+    // Formatting.
+    case InputEvent::InputType::kFormatBold:
+    case InputEvent::InputType::kFormatItalic:
+    case InputEvent::InputType::kFormatUnderline:
+    case InputEvent::InputType::kFormatStrikeThrough:
+    case InputEvent::InputType::kFormatSuperscript:
+    case InputEvent::InputType::kFormatSubscript:
+    case InputEvent::InputType::kFormatJustifyCenter:
+    case InputEvent::InputType::kFormatJustifyFull:
+    case InputEvent::InputType::kFormatJustifyRight:
+    case InputEvent::InputType::kFormatJustifyLeft:
+    case InputEvent::InputType::kFormatIndent:
+    case InputEvent::InputType::kFormatOutdent:
+    case InputEvent::InputType::kFormatRemove:
+    case InputEvent::InputType::kFormatSetBlockTextDirection:
+      command = ax::mojom::blink::Command::kFormat;
+      break;
+
+    case InputEvent::InputType::kNumberOfInputTypes:
+      NOTREACHED() << "Should never be assigned as an input type.";
+      command = ax::mojom::blink::Command::kType;
+      break;
+  }
+
+  return BlinkAXEventIntent(command, text_boundary, move_direction);
+}
+
+// static
 BlinkAXEventIntent BlinkAXEventIntent::FromClearedSelection(
     const SetSelectionBy set_selection_by) {
   // |text_boundary| and |move_direction| are not used in this case.
@@ -19,6 +143,7 @@ BlinkAXEventIntent BlinkAXEventIntent::FromClearedSelection(
                             ax::mojom::blink::MoveDirection::kForward);
 }
 
+// static
 BlinkAXEventIntent BlinkAXEventIntent::FromModifiedSelection(
     const SelectionModifyAlteration alter,
     const SelectionModifyDirection direction,
@@ -145,6 +270,7 @@ BlinkAXEventIntent BlinkAXEventIntent::FromModifiedSelection(
   return BlinkAXEventIntent(command, text_boundary, move_direction);
 }
 
+// static
 BlinkAXEventIntent BlinkAXEventIntent::FromNewSelection(
     const TextGranularity granularity,
     bool is_base_first,

third_party/blink/renderer/core/accessibility/blink_ax_event_intent.h

@@ -8,6 +8,7 @@
 #include <string>
 
 #include "third_party/blink/renderer/core/core_export.h"
+#include "third_party/blink/renderer/core/editing/commands/edit_command.h"
 #include "third_party/blink/renderer/core/editing/selection_modifier.h"
 #include "third_party/blink/renderer/core/editing/set_selection_options.h"
 #include "third_party/blink/renderer/core/editing/text_granularity.h"
@@ -29,6 +30,7 @@ namespace blink {
 // previous word, or it could have been moved to the end of the next line.
 class CORE_EXPORT BlinkAXEventIntent final {
  public:
+  static BlinkAXEventIntent FromEditCommand(const EditCommand& edit_command);
   static BlinkAXEventIntent FromClearedSelection(
       const SetSelectionBy set_selection_by);
   static BlinkAXEventIntent FromModifiedSelection(

third_party/blink/renderer/core/editing/commands/composite_edit_command.cc

@@ -26,6 +26,9 @@
 #include "third_party/blink/renderer/core/editing/commands/composite_edit_command.h"
 
 #include <algorithm>
+
+#include "third_party/blink/renderer/core/accessibility/blink_ax_event_intent.h"
+#include "third_party/blink/renderer/core/accessibility/scoped_blink_ax_event_intent.h"
 #include "third_party/blink/renderer/core/dom/document.h"
 #include "third_party/blink/renderer/core/dom/document_fragment.h"
 #include "third_party/blink/renderer/core/dom/element_traversal.h"
@@ -153,6 +156,11 @@ bool CompositeEditCommand::Apply() {
   SetSelectionIsDirectional(frame->Selection().IsDirectional());
   GetUndoStep()->SetSelectionIsDirectional(SelectionIsDirectional());
 
+  // Provides details to accessibility about any text change caused by applying
+  // this command, throughout the current call stack.
+  ScopedBlinkAXEventIntent scoped_blink_ax_event_intent(
+      BlinkAXEventIntent::FromEditCommand(*this), &GetDocument());
+
   EditingState editing_state;
   EventQueueScope event_queue_scope;
   DoApply(&editing_state);

third_party/blink/web_tests/accessibility/contenteditable-notifications.html

@@ -15,13 +15,17 @@ async_test((t) => {
   const axTextBox = accessibilityController.accessibleElementById('textbox');
   let valueChangedCount = 0;
   let selectedTextChangedCount = 0;
-  const expectedIntents = [];
+  const expectedValueChangedIntents = [];
+  const expectedSelectedTextChangedIntents = [];
 
   axTextBox.setNotificationListener(t.step_func((notification, intents) => {
     if (notification == "ValueChanged") {
+      assert_array_equals(intents,
+                          expectedValueChangedIntents[valueChangedCount]);
       ++valueChangedCount;
     } else if (notification == "SelectedTextChanged") {
-      assert_array_equals(intents, [expectedIntents[selectedTextChangedCount]]);
+      assert_array_equals(intents,
+                          expectedSelectedTextChangedIntents[selectedTextChangedCount]);
       ++selectedTextChangedCount;
     }
 
@@ -30,28 +34,67 @@ async_test((t) => {
     }
   }));
 
-  runAfterLayoutAndPaint(t.step_func(() => {
-    textbox.focus();
-    // Initial setting of the selection.
-    expectedIntents.push('AXEventIntent(setSelection,character,forward)');
-    eventSender.keyDown("ArrowDown", []);
-    expectedIntents.push('AXEventIntent(moveSelection,lineStart,forward)');
-    eventSender.keyDown("ArrowDown", []);
-    expectedIntents.push('AXEventIntent(moveSelection,lineStart,forward)');
-    eventSender.keyDown("ArrowLeft", []);
-    expectedIntents.push('AXEventIntent(moveSelection,character,backward)');
-    eventSender.keyDown("ArrowLeft", []);
-    expectedIntents.push('AXEventIntent(moveSelection,character,backward)');
-    eventSender.keyDown("w", []);
-    // TODO(nektar): Support an intent for the typing command.
-    expectedIntents.push('AXEventIntent(setSelection,character,forward)');
-    eventSender.keyDown("x", []);
-    expectedIntents.push('AXEventIntent(setSelection,character,forward)');
-    eventSender.keyDown("y", []);
-    expectedIntents.push('AXEventIntent(setSelection,character,forward)');
-    eventSender.keyDown("z", []);
-    expectedIntents.push('AXEventIntent(setSelection,character,forward)');
-  }));
-}, 'Moving the cursor in a contentEditable sends a selected text change notification, and typing in a contentEditable sends both a value changed and selected text changed notification.');
+  // Initial setting of the selection.
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(setSelection,character,forward)'
+  ]);
+  textbox.focus();
+
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(moveSelection,lineStart,forward)'
+  ]);
+  eventSender.keyDown("ArrowDown", []);
+
+  expectedValueChangedIntents.push([
+    'AXEventIntent(type,character,forward)'
+  ]);
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(setSelection,character,forward)',
+    'AXEventIntent(type,character,forward)'
+  ]);
+  eventSender.keyDown("w", []);
+
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(moveSelection,lineStart,forward)'
+  ]);
+  eventSender.keyDown("ArrowDown", []);
+
+  expectedValueChangedIntents.push([
+    'AXEventIntent(type,character,forward)'
+  ]);
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(type,character,forward)',
+    'AXEventIntent(setSelection,character,forward)'
+  ]);
+  eventSender.keyDown("x", []);
+
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(moveSelection,character,backward)'
+  ]);
+  eventSender.keyDown("ArrowLeft", []);
+
+  expectedValueChangedIntents.push([
+    'AXEventIntent(type,character,forward)'
+  ]);
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(setSelection,character,forward)',
+    'AXEventIntent(type,character,forward)'
+  ]);
+  eventSender.keyDown("y", []);
+
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(moveSelection,character,backward)'
+  ]);
+  eventSender.keyDown("ArrowLeft", []);
+
+  expectedValueChangedIntents.push([
+    'AXEventIntent(type,character,forward)'
+  ]);
+  expectedSelectedTextChangedIntents.push([
+    'AXEventIntent(setSelection,character,forward)',
+    'AXEventIntent(type,character,forward)'
+  ]);
+  eventSender.keyDown("z", []);
+}, 'This test ensures that moving the cursor in a contentEditable sends a selected text change notification, and typing in a contentEditable sends both a value changed and selected text changed notification.');
 
 </script>