[Switch Access] Point Scan infrastructure

This change sets up the infrastructure for future development of an
alternate form of selection via Switch Access called Point Scanning.
(go/cros-switch-point-scanning has more details).

Specifically, this change creates a PointScanController to drive the
logic and a PointScanLayer to draw the UI, as well as a command-line
to develop behind. Currently, if the flag
kEnableSwitchAccessPointScanning is enabled, Switch Access draws a
vertical line through the center of the screen when the feature is
turned on.

Bug: 1061537
Change-Id: Ifba390904d5e94ee5edbe494a6799168a95a7272
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2092731
Commit-Queue: Anastasia Helfinstein <anastasi@google.com>
Reviewed-by: David Tseng <dtseng@chromium.org>
Cr-Commit-Position: refs/heads/master@{#786903}

ash/BUILD.gn

@@ -159,6 +159,10 @@ component("ash") {
     "accessibility/key_accessibility_enabler.h",
     "accessibility/layer_animation_info.cc",
     "accessibility/layer_animation_info.h",
+    "accessibility/point_scan_controller.cc",
+    "accessibility/point_scan_controller.h",
+    "accessibility/point_scan_layer.cc",
+    "accessibility/point_scan_layer.h",
     "accessibility/spoken_feedback_enabler.cc",
     "accessibility/spoken_feedback_enabler.h",
     "accessibility/touch_accessibility_enabler.cc",
@@ -1821,6 +1825,7 @@ test("ash_unittests") {
     "accessibility/key_accessibility_enabler_unittest.cc",
     "accessibility/mock_touch_exploration_controller_delegate.cc",
     "accessibility/mock_touch_exploration_controller_delegate.h",
+    "accessibility/point_scan_controller_unittest.cc",
     "accessibility/touch_accessibility_enabler_unittest.cc",
     "accessibility/touch_exploration_controller_unittest.cc",
     "accessibility/touch_exploration_manager_unittest.cc",

ash/accessibility/accessibility_controller_impl.cc

@@ -13,6 +13,7 @@
 #include "ash/accessibility/accessibility_highlight_controller.h"
 #include "ash/accessibility/accessibility_observer.h"
 #include "ash/accessibility/accessibility_panel_layout_manager.h"
+#include "ash/accessibility/point_scan_controller.h"
 #include "ash/autoclick/autoclick_controller.h"
 #include "ash/events/select_to_speak_event_handler.h"
 #include "ash/events/switch_access_event_handler.h"
@@ -1169,6 +1170,13 @@ void AccessibilityControllerImpl::ForwardKeyEventsToSwitchAccess(
   switch_access_event_handler_->set_forward_key_events(should_forward);
 }
 
+void AccessibilityControllerImpl::StartPointScanning() {
+  if (!point_scan_controller_)
+    point_scan_controller_.reset(new PointScanController());
+
+  point_scan_controller_->Start();
+}
+
 void AccessibilityControllerImpl::SetSwitchAccessEventHandlerDelegate(
     SwitchAccessEventHandlerDelegate* delegate) {
   switch_access_event_handler_delegate_ = delegate;
@@ -1975,6 +1983,11 @@ void AccessibilityControllerImpl::UpdateFeatureFromPref(FeatureType feature) {
         switch_access_bubble_controller_ =
             std::make_unique<SwitchAccessMenuBubbleController>();
         MaybeCreateSwitchAccessEventHandler();
+
+        if (::switches::IsSwitchAccessPointScanningEnabled()) {
+          StartPointScanning();
+        }
+
         ShowAccessibilityNotification(
             A11yNotificationType::kSwitchAccessEnabled);
       }

ash/accessibility/accessibility_controller_impl.h

@@ -39,6 +39,7 @@ namespace ash {
 class AccessibilityHighlightController;
 class AccessibilityObserver;
 class FloatingAccessibilityController;
+class PointScanController;
 class ScopedBacklightsForcedOff;
 class SelectToSpeakEventHandler;
 class SwitchAccessMenuBubbleController;
@@ -370,6 +371,10 @@ class ASH_EXPORT AccessibilityControllerImpl : public AccessibilityController,
   // accessibility tray menu.
   bool IsAdditionalSettingsSeparatorVisibleInTray();
 
+  // Starts point scanning, to select a point onscreen without using a mouse
+  // (as used by Switch Access).
+  void StartPointScanning();
+
   // AccessibilityController:
   void SetClient(AccessibilityControllerClient* client) override;
   void SetDarkenScreen(bool darken) override;
@@ -492,6 +497,10 @@ class ASH_EXPORT AccessibilityControllerImpl : public AccessibilityController,
   // makes floating menu visible as soon as it is enabled.
   bool always_show_floating_menu_when_enabled_ = false;
 
+  // Used to control point scanning, or selecting a point onscreen without using
+  // a mouse (as done by Switch Access).
+  std::unique_ptr<PointScanController> point_scan_controller_;
+
   // Used to force the backlights off to darken the screen.
   std::unique_ptr<ScopedBacklightsForcedOff> scoped_backlights_forced_off_;
 

ash/accessibility/point_scan_controller.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ash/accessibility/point_scan_controller.h"
+
+#include "ash/accessibility/point_scan_layer.h"
+#include "ui/display/display.h"
+#include "ui/display/screen.h"
+
+namespace ash {
+
+PointScanController::PointScanController() = default;
+PointScanController::~PointScanController() = default;
+
+void PointScanController::Start() {
+  point_scan_layer_.reset(new PointScanLayer(this));
+  point_scan_layer_->StartHorizontalScanning();
+}
+
+void PointScanController::OnDeviceScaleFactorChanged() {}
+void PointScanController::OnAnimationStep(base::TimeTicks timestamp) {}
+
+}  // namespace ash

ash/accessibility/point_scan_controller.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ASH_ACCESSIBILITY_POINT_SCAN_CONTROLLER_H_
+#define ASH_ACCESSIBILITY_POINT_SCAN_CONTROLLER_H_
+
+#include "ash/accessibility/accessibility_layer.h"
+#include "ash/ash_export.h"
+
+namespace ash {
+
+class PointScanLayer;
+
+// PointScanController handles drawing and animating custom lines onscreen, for
+// the purposes of selecting a point onscreen without using a traditional mouse
+// or keyboard. Currently used by Switch Access.
+class ASH_EXPORT PointScanController : public AccessibilityLayerDelegate {
+ public:
+  PointScanController();
+  ~PointScanController() override;
+
+  PointScanController(const PointScanController&) = delete;
+  PointScanController& operator=(const PointScanController&) = delete;
+
+  // Starts point scanning, by sweeping a line across the screen and waiting for
+  // user input.
+  // TODO(crbug/1061537): Animate the line across the screen.
+  void Start();
+
+ private:
+  // AccessibilityLayerDelegate implementation:
+  void OnDeviceScaleFactorChanged() override;
+  void OnAnimationStep(base::TimeTicks timestamp) override;
+
+  std::unique_ptr<PointScanLayer> point_scan_layer_;
+};
+
+}  // namespace ash
+
+#endif  // ASH_ACCESSIBILITY_POINT_SCAN_CONTROLLER_H_

ash/accessibility/point_scan_controller_unittest.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ash/accessibility/point_scan_controller.h"
+
+#include "ash/shell.h"
+#include "ash/test/ash_test_base.h"
+#include "base/bind.h"
+#include "base/command_line.h"
+#include "base/macros.h"
+#include "third_party/skia/include/core/SkBitmap.h"
+#include "ui/accessibility/accessibility_switches.h"
+#include "ui/aura/window.h"
+#include "ui/compositor/compositor_switches.h"
+#include "ui/gfx/image/image.h"
+#include "ui/snapshot/snapshot.h"
+
+namespace ash {
+
+class PointScanControllerTest : public AshTestBase {
+ protected:
+  PointScanControllerTest() = default;
+  ~PointScanControllerTest() override = default;
+
+  // AshTestBase:
+  void SetUp() override {
+    base::CommandLine::ForCurrentProcess()->AppendSwitch(
+        ::switches::kEnableSwitchAccessPointScanning);
+    base::CommandLine::ForCurrentProcess()->AppendSwitch(
+        ::switches::kEnablePixelOutputInTests);
+    AshTestBase::SetUp();
+  }
+
+  void CaptureBeforeImage(const gfx::Rect& bounds) {
+    Capture(bounds);
+    if (before_bmp_.tryAllocPixels(image_.AsBitmap().info())) {
+      image_.AsBitmap().readPixels(before_bmp_.info(), before_bmp_.getPixels(),
+                                   before_bmp_.rowBytes(), 0, 0);
+    }
+  }
+
+  void CaptureAfterImage(const gfx::Rect& bounds) {
+    Capture(bounds);
+    if (after_bmp_.tryAllocPixels(image_.AsBitmap().info())) {
+      image_.AsBitmap().readPixels(after_bmp_.info(), after_bmp_.getPixels(),
+                                   after_bmp_.rowBytes(), 0, 0);
+    }
+  }
+
+  void Capture(const gfx::Rect& bounds) {
+    // Occasionally we don't get any pixels the first try.
+    // Keep trying until we get the correct size bitmap and the first pixel
+    // is transparent.
+    while (true) {
+      aura::Window* window = Shell::GetPrimaryRootWindow();
+      base::RunLoop run_loop;
+      ui::GrabWindowSnapshotAndScaleAsync(
+          window, bounds, bounds.size(),
+          base::BindOnce(
+              [](base::RunLoop* run_loop, gfx::Image* image,
+                 gfx::Image got_image) {
+                run_loop->Quit();
+                *image = got_image;
+              },
+              &run_loop, &image_));
+      run_loop.Run();
+      SkBitmap bitmap = image_.AsBitmap();
+      if (bitmap.width() != bounds.width() ||
+          bitmap.height() != bounds.height()) {
+        LOG(INFO) << "Bitmap not correct size, trying to capture again";
+        continue;
+      } else if (255 == SkColorGetA(bitmap.getColor(0, 0))) {
+        LOG(INFO) << "Bitmap is transparent, trying to capture again";
+        break;
+      }
+    }
+  }
+
+  void ComputeImageStats() {
+    diff_count_ = 0;
+    row_diff_count_ = 0;
+    col_diff_count_ = 0;
+    bool row_diff_tracker_[before_bmp_.height()];
+    for (int i = 0; i < before_bmp_.height(); ++i) {
+      row_diff_tracker_[i] = false;
+    }
+
+    for (int x = 0; x < before_bmp_.width(); ++x) {
+      bool col_diff = false;
+      for (int y = 0; y < before_bmp_.height(); ++y) {
+        SkColor before_color = before_bmp_.getColor(x, y);
+        SkColor after_color = after_bmp_.getColor(x, y);
+        if (before_color != after_color) {
+          diff_count_++;
+          col_diff = true;
+          row_diff_tracker_[y] = true;
+        }
+      }
+      if (col_diff)
+        ++col_diff_count_;
+    }
+
+    for (int i = 0; i < before_bmp_.height(); ++i) {
+      if (row_diff_tracker_[i])
+        ++row_diff_count_;
+    }
+  }
+
+  int diff_count() const { return diff_count_; }
+  int row_diff_count() const { return row_diff_count_; }
+  int col_diff_count() const { return col_diff_count_; }
+
+ private:
+  gfx::Image image_;
+  SkBitmap before_bmp_;
+  SkBitmap after_bmp_;
+  int diff_count_ = 0;
+  int row_diff_count_ = 0;
+  int col_diff_count_ = 0;
+
+  DISALLOW_COPY_AND_ASSIGN(PointScanControllerTest);
+};
+
+TEST_F(PointScanControllerTest, StartScanning) {
+  gfx::Rect bounds = Shell::GetPrimaryRootWindow()->bounds();
+
+  // Create a white background window for captured image color smoke test.
+  CreateTestWindowInShell(SK_ColorWHITE, -1, bounds);
+
+  // For some reason, the white window is not fully drawn the first time we call
+  // CaptureBeforeImage.
+  CaptureBeforeImage(bounds);
+  CaptureBeforeImage(bounds);
+
+  PointScanController controller;
+  controller.Start();
+
+  CaptureAfterImage(bounds);
+  ComputeImageStats();
+
+  EXPECT_EQ(3600, diff_count());
+  EXPECT_EQ(6, col_diff_count());
+  EXPECT_EQ(600, row_diff_count());
+}
+
+}  // namespace ash

ash/accessibility/point_scan_layer.cc

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "ash/accessibility/point_scan_layer.h"
+
+#include "ash/shell.h"
+#include "third_party/skia/include/core/SkPaint.h"
+#include "third_party/skia/include/core/SkPath.h"
+#include "ui/aura/window.h"
+#include "ui/compositor/layer.h"
+#include "ui/compositor/paint_recorder.h"
+#include "ui/display/display.h"
+#include "ui/display/screen.h"
+#include "ui/gfx/canvas.h"
+#include "ui/gfx/color_palette.h"
+#include "ui/wm/core/coordinate_conversion.h"
+
+namespace ash {
+
+namespace {
+const int kDefaultStrokeWidth = 6;
+
+display::Display GetPrimaryDisplay() {
+  DCHECK(display::Screen::GetScreen());
+  return display::Screen::GetScreen()->GetPrimaryDisplay();
+}
+}  // namespace
+
+PointScanLayer::PointScanLayer(AccessibilityLayerDelegate* delegate)
+    : AccessibilityLayer(delegate) {
+  aura::Window* root_window =
+      Shell::GetRootWindowForDisplayId(GetPrimaryDisplay().id());
+  CreateOrUpdateLayer(root_window, "PointScanning", gfx::Rect());
+  SetOpacity(1.0);
+}
+
+void PointScanLayer::StartHorizontalScanning() {
+  bounds_ = GetPrimaryDisplay().bounds();
+  layer()->SetBounds(bounds_);
+
+  gfx::Point start = bounds_.top_center();
+  gfx::Point end = bounds_.bottom_center();
+  horizontal_.start = start;
+  horizontal_.end = end;
+}
+
+bool PointScanLayer::CanAnimate() const {
+  return true;
+}
+bool PointScanLayer::NeedToAnimate() const {
+  return true;
+}
+int PointScanLayer::GetInset() const {
+  return 0;
+}
+
+void PointScanLayer::OnPaintLayer(const ui::PaintContext& context) {
+  ui::PaintRecorder recorder(context, layer()->size());
+  cc::PaintFlags flags;
+  flags.setAntiAlias(true);
+  flags.setStyle(cc::PaintFlags::kStroke_Style);
+  flags.setStrokeWidth(kDefaultStrokeWidth);
+  flags.setColor(gfx::kGoogleBlue300);
+
+  SkPath path;
+  path.moveTo(horizontal_.start.x(), horizontal_.start.y());
+  path.lineTo(horizontal_.end.x(), horizontal_.end.y());
+  recorder.canvas()->DrawPath(path, flags);
+}
+
+}  // namespace ash

ash/accessibility/point_scan_layer.h

+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef ASH_ACCESSIBILITY_POINT_SCAN_LAYER_H_
+#define ASH_ACCESSIBILITY_POINT_SCAN_LAYER_H_
+
+#include "ash/accessibility/accessibility_layer.h"
+#include "ash/ash_export.h"
+
+namespace ash {
+
+class PointScanLayer : public AccessibilityLayer {
+ public:
+  explicit PointScanLayer(AccessibilityLayerDelegate* delegate);
+  ~PointScanLayer() override = default;
+
+  PointScanLayer(const PointScanLayer&) = delete;
+  PointScanLayer& operator=(const PointScanLayer&) = delete;
+
+  // Begins sweeping a line horizontally across the screen, for the user to pick
+  // an x-coordinate.
+  // TODO(crbug/1061537): Animate the line across the screen.
+  void StartHorizontalScanning();
+
+  // AccessibilityLayer overrides:
+  bool CanAnimate() const override;
+  bool NeedToAnimate() const override;
+  int GetInset() const override;
+
+ private:
+  // ui:LayerDelegate overrides:
+  void OnPaintLayer(const ui::PaintContext& context) override;
+
+  struct Line {
+    gfx::Point start;
+    gfx::Point end;
+  };
+
+  // The bounds within which we are scanning.
+  gfx::Rect bounds_;
+
+  // The line currently being drawn.
+  Line horizontal_;
+};
+
+}  // namespace ash
+
+#endif  // ASH_ACCESSIBILITY_POINT_SCAN_LAYER_H_

ui/accessibility/accessibility_switches.cc

@@ -52,6 +52,10 @@ const char kDisableExperimentalAccessibilityChromeVoxSearchMenus[] =
 const char kEnableExperimentalAccessibilityChromeVoxTutorial[] =
     "enable-experimental-accessibility-chromevox-tutorial";
 
+// Enables Switch Access point scanning. This feature hasn't launched yet.
+const char kEnableSwitchAccessPointScanning[] =
+    "enable-switch-access-point-scanning";
+
 bool IsExperimentalAccessibilityLanguageDetectionEnabled() {
   return base::CommandLine::ForCurrentProcess()->HasSwitch(
       ::switches::kEnableExperimentalAccessibilityLanguageDetection);
@@ -67,6 +71,11 @@ bool IsExperimentalAccessibilitySwitchAccessTextEnabled() {
       ::switches::kEnableExperimentalAccessibilitySwitchAccessText);
 }
 
+bool IsSwitchAccessPointScanningEnabled() {
+  return base::CommandLine::ForCurrentProcess()->HasSwitch(
+      ::switches::kEnableSwitchAccessPointScanning);
+}
+
 #if defined(OS_WIN)
 // Enables UI Automation platform API in addition to the IAccessible API.
 const char kEnableExperimentalUIAutomation[] =

ui/accessibility/accessibility_switches.h

@@ -29,6 +29,7 @@ AX_BASE_EXPORT extern const char
     kDisableExperimentalAccessibilityChromeVoxSearchMenus[];
 AX_BASE_EXPORT extern const char
     kEnableExperimentalAccessibilityChromeVoxTutorial[];
+AX_BASE_EXPORT extern const char kEnableSwitchAccessPointScanning[];
 
 // Returns true if experimental accessibility language detection is enabled.
 AX_BASE_EXPORT bool IsExperimentalAccessibilityLanguageDetectionEnabled();
@@ -48,6 +49,9 @@ AX_BASE_EXPORT extern const char kEnableExperimentalUIAutomation[];
 // Returns true if experimental support for UIAutomation is enabled.
 AX_BASE_EXPORT bool IsExperimentalAccessibilityPlatformUIAEnabled();
 
+// Returns true if Switch Access point scanning is enabled.
+AX_BASE_EXPORT bool IsSwitchAccessPointScanningEnabled();
+
 }  // namespace switches
 
 #endif  // UI_ACCESSIBILITY_ACCESSIBILITY_SWITCHES_H_