Commit 35642255 authored by Mark Schillaci's avatar Mark Schillaci Committed by Commit Bot

Accessibility Image Descriptions Android Port, Part III - Integration 1

This CL is the third part of an effort to port the existing Desktop
feature to Android. This feature enables a user to send an image to
Google to process to generate a descriptive alt text if a website
does not provide one.

Design Doc: go/2020-q1-android-image-descriptions
Slide Deck: go/clank-imageDescriptions
Launch Bug: 1057168

Original Desktop Design Doc for reference:
go/chrome-a11y-annotations-design

This is a conservative approach, we use a separate set of profile
Prefs for this feature rather than syncing with the Desktop Prefs.
This can be updated in time as needed. We have also hidden the
entire feature behind a feature flag, so this CL is not visible
to an average user.

----------

This CL adds the following:

- Adds helper text/hint for unlabeled images
- Adds returned image description to node's inner text
- Associated strings etc for above

----------

Binary-Size: Size increase is unavoidable, it comes from adding necessary content strings and descriptions for translations.
AX-Relnotes: Adds feature to enable generating alt image descriptions, currently behind a feature flag.
Bug: 1057169
Change-Id: I6becb973045e127681136a1fe0b7171d5cbddca8
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2317712Reviewed-by: default avatardanakj <danakj@chromium.org>
Reviewed-by: default avatarMark Schillaci <mschillaci@google.com>
Reviewed-by: default avatarDominic Mazzoni <dmazzoni@chromium.org>
Commit-Queue: Mark Schillaci <mschillaci@google.com>
Cr-Commit-Position: refs/heads/master@{#821324}
parent 8698e421
......@@ -91,6 +91,50 @@ void BrowserAccessibilityAndroid::OnLocationChanged() {
manager->FireLocationChanged(this);
}
base::string16
BrowserAccessibilityAndroid::GetLocalizedStringForImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus status) const {
// Default to standard text, except for special case of eligible.
if (status != ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation)
return BrowserAccessibility::GetLocalizedStringForImageAnnotationStatus(
status);
ContentClient* content_client = content::GetContentClient();
int message_id = 0;
switch (static_cast<ax::mojom::WritingDirection>(
GetIntAttribute(ax::mojom::IntAttribute::kTextDirection))) {
case ax::mojom::WritingDirection::kRtl:
message_id = IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION_ANDROID_RTL;
break;
case ax::mojom::WritingDirection::kTtb:
case ax::mojom::WritingDirection::kBtt:
case ax::mojom::WritingDirection::kNone:
case ax::mojom::WritingDirection::kLtr:
message_id = IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION_ANDROID_LTR;
break;
}
DCHECK(message_id);
return content_client->GetLocalizedString(message_id);
}
void BrowserAccessibilityAndroid::AppendTextToString(
base::string16 extra_text,
base::string16* string) const {
if (extra_text.empty())
return;
if (string->empty()) {
*string = extra_text;
return;
}
*string += base::string16(base::ASCIIToUTF16(", ")) + extra_text;
}
bool BrowserAccessibilityAndroid::IsCheckable() const {
return GetData().HasCheckedState();
}
......@@ -479,6 +523,31 @@ base::string16 BrowserAccessibilityAndroid::GetInnerText() const {
if (GetRole() == ax::mojom::Role::kRootWebArea)
return text;
// Append image description strings to the text.
auto status = GetData().GetImageAnnotationStatus();
switch (status) {
case ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation:
case ax::mojom::ImageAnnotationStatus::kAnnotationPending:
case ax::mojom::ImageAnnotationStatus::kAnnotationEmpty:
case ax::mojom::ImageAnnotationStatus::kAnnotationAdult:
case ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed:
AppendTextToString(GetLocalizedStringForImageAnnotationStatus(status),
&text);
break;
case ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded:
AppendTextToString(
GetString16Attribute(ax::mojom::StringAttribute::kImageAnnotation),
&text);
break;
case ax::mojom::ImageAnnotationStatus::kNone:
case ax::mojom::ImageAnnotationStatus::kWillNotAnnotateDueToScheme:
case ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation:
case ax::mojom::ImageAnnotationStatus::kSilentlyEligibleForAnnotation:
break;
}
// This is called from IsLeaf, so don't call PlatformChildCount
// from within this!
if (text.empty() && (HasOnlyTextChildren() ||
......@@ -756,6 +825,24 @@ base::string16 BrowserAccessibilityAndroid::GetRoleDescription() const {
return parent->GetRoleDescription();
}
// If this node is an image, check status and potentially add unlabeled role.
auto status = GetData().GetImageAnnotationStatus();
switch (status) {
case ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation:
case ax::mojom::ImageAnnotationStatus::kAnnotationPending:
case ax::mojom::ImageAnnotationStatus::kAnnotationEmpty:
case ax::mojom::ImageAnnotationStatus::kAnnotationAdult:
case ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed:
return GetLocalizedRoleDescriptionForUnlabeledImage();
case ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded:
case ax::mojom::ImageAnnotationStatus::kNone:
case ax::mojom::ImageAnnotationStatus::kWillNotAnnotateDueToScheme:
case ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation:
case ax::mojom::ImageAnnotationStatus::kSilentlyEligibleForAnnotation:
break;
}
int message_id = -1;
switch (GetRole()) {
case ax::mojom::Role::kAbbr:
......
......@@ -26,6 +26,8 @@ class CONTENT_EXPORT BrowserAccessibilityAndroid : public BrowserAccessibility {
// Overrides from BrowserAccessibility.
void OnDataChanged() override;
void OnLocationChanged() override;
base::string16 GetLocalizedStringForImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus status) const override;
bool IsCheckable() const;
bool IsChecked() const;
......@@ -195,6 +197,9 @@ class CONTENT_EXPORT BrowserAccessibilityAndroid : public BrowserAccessibility {
static size_t CommonEndLengths(const base::string16 a,
const base::string16 b);
void AppendTextToString(base::string16 extra_text,
base::string16* string) const;
base::string16 cached_text_;
base::string16 old_value_;
base::string16 new_value_;
......
......@@ -10,10 +10,39 @@
#include "build/build_config.h"
#include "content/browser/accessibility/browser_accessibility_manager.h"
#include "content/browser/accessibility/test_browser_accessibility_delegate.h"
#include "content/test/test_content_client.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "third_party/blink/public/strings/grit/blink_strings.h"
namespace content {
class MockContentClient : public TestContentClient {
public:
base::string16 GetLocalizedString(int message_id) override {
switch (message_id) {
case IDS_AX_UNLABELED_IMAGE_ROLE_DESCRIPTION:
return base::ASCIIToUTF16("Unlabeled image");
case IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION_ANDROID_LTR:
return base::ASCIIToUTF16(
"This image isn't labeled. Open the More Options menu at the top "
"right to get image descriptions.");
case IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION_ANDROID_RTL:
return base::ASCIIToUTF16(
"This image isn't labeled. Open the More Options menu at the top "
"left to get image descriptions.");
case IDS_AX_IMAGE_ANNOTATION_PENDING:
return base::ASCIIToUTF16("Getting description...");
case IDS_AX_IMAGE_ANNOTATION_ADULT:
return base::ASCIIToUTF16(
"Appears to contain adult content. No description available.");
case IDS_AX_IMAGE_ANNOTATION_NO_DESCRIPTION:
return base::ASCIIToUTF16("No description available.");
default:
return base::string16();
}
}
};
class BrowserAccessibilityAndroidTest : public testing::Test {
public:
BrowserAccessibilityAndroidTest();
......@@ -26,6 +55,7 @@ class BrowserAccessibilityAndroidTest : public testing::Test {
private:
void SetUp() override;
base::test::TaskEnvironment task_environment_;
MockContentClient client_;
DISALLOW_COPY_AND_ASSIGN(BrowserAccessibilityAndroidTest);
};
......@@ -37,6 +67,7 @@ void BrowserAccessibilityAndroidTest::SetUp() {
ui::AXPlatformNode::NotifyAddAXModeFlags(ui::kAXModeComplete);
test_browser_accessibility_delegate_ =
std::make_unique<TestBrowserAccessibilityDelegate>();
SetContentClient(&client_);
}
TEST_F(BrowserAccessibilityAndroidTest, TestRetargetTextOnly) {
......@@ -280,4 +311,301 @@ TEST_F(BrowserAccessibilityAndroidTest, TestRetargetInputControl) {
manager.reset();
}
TEST_F(BrowserAccessibilityAndroidTest,
TestImageRoleDescription_UnlabeledImage) {
ui::AXTreeUpdate tree;
tree.root_id = 1;
tree.nodes.resize(6);
tree.nodes[0].id = 1;
tree.nodes[0].child_ids = {2, 3, 4, 5, 6};
// Images with these annotation statuses should report "Unlabeled image"
tree.nodes[1].id = 2;
tree.nodes[1].role = ax::mojom::Role::kImage;
tree.nodes[1].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation);
tree.nodes[2].id = 3;
tree.nodes[2].role = ax::mojom::Role::kImage;
tree.nodes[2].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationPending);
tree.nodes[3].id = 4;
tree.nodes[3].role = ax::mojom::Role::kImage;
tree.nodes[3].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationEmpty);
tree.nodes[4].id = 5;
tree.nodes[4].role = ax::mojom::Role::kImage;
tree.nodes[4].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationAdult);
tree.nodes[5].id = 6;
tree.nodes[5].role = ax::mojom::Role::kImage;
tree.nodes[5].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed);
std::unique_ptr<BrowserAccessibilityManager> manager(
BrowserAccessibilityManager::Create(
tree, test_browser_accessibility_delegate_.get()));
for (int child_index = 0; child_index < int{tree.nodes[0].child_ids.size()};
++child_index) {
BrowserAccessibilityAndroid* child =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(child_index));
EXPECT_EQ(base::ASCIIToUTF16("Unlabeled image"),
child->GetRoleDescription());
}
}
TEST_F(BrowserAccessibilityAndroidTest, TestImageRoleDescription_Empty) {
ui::AXTreeUpdate tree;
tree.root_id = 1;
tree.nodes.resize(6);
tree.nodes[0].id = 1;
tree.nodes[0].child_ids = {2, 3, 4, 5, 6};
// Images with these annotation statuses should report nothing.
tree.nodes[1].id = 2;
tree.nodes[1].role = ax::mojom::Role::kImage;
tree.nodes[1].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded);
tree.nodes[2].id = 3;
tree.nodes[2].role = ax::mojom::Role::kImage;
tree.nodes[2].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kNone);
tree.nodes[3].id = 4;
tree.nodes[3].role = ax::mojom::Role::kImage;
tree.nodes[3].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kWillNotAnnotateDueToScheme);
tree.nodes[4].id = 5;
tree.nodes[4].role = ax::mojom::Role::kImage;
tree.nodes[4].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation);
tree.nodes[5].id = 6;
tree.nodes[5].role = ax::mojom::Role::kImage;
tree.nodes[5].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kSilentlyEligibleForAnnotation);
std::unique_ptr<BrowserAccessibilityManager> manager(
BrowserAccessibilityManager::Create(
tree, test_browser_accessibility_delegate_.get()));
for (int child_index = 0; child_index < int{tree.nodes[0].child_ids.size()};
++child_index) {
BrowserAccessibilityAndroid* child =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(child_index));
EXPECT_EQ(base::string16(), child->GetRoleDescription());
}
}
TEST_F(BrowserAccessibilityAndroidTest, TestImageInnerText_Eligible) {
ui::AXTreeUpdate tree;
tree.root_id = 1;
tree.nodes.resize(3);
tree.nodes[0].id = 1;
tree.nodes[0].child_ids = {2, 3};
tree.nodes[1].id = 2;
tree.nodes[1].role = ax::mojom::Role::kImage;
tree.nodes[1].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation);
tree.nodes[1].AddIntAttribute(
ax::mojom::IntAttribute::kTextDirection,
static_cast<int32_t>(ax::mojom::WritingDirection::kLtr));
tree.nodes[2].id = 3;
tree.nodes[2].role = ax::mojom::Role::kImage;
tree.nodes[2].SetName("image_name");
tree.nodes[2].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation);
tree.nodes[2].AddIntAttribute(
ax::mojom::IntAttribute::kTextDirection,
static_cast<int32_t>(ax::mojom::WritingDirection::kRtl));
std::unique_ptr<BrowserAccessibilityManager> manager(
BrowserAccessibilityManager::Create(
tree, test_browser_accessibility_delegate_.get()));
BrowserAccessibilityAndroid* image_ltr =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(0));
EXPECT_EQ(
base::ASCIIToUTF16("This image isn't labeled. Open the More Options menu "
"at the top right to get image descriptions."),
image_ltr->GetInnerText());
BrowserAccessibilityAndroid* image_rtl =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(1));
EXPECT_EQ(base::ASCIIToUTF16(
"image_name, This image isn't labeled. Open the More Options "
"menu at the top left to get image descriptions."),
image_rtl->GetInnerText());
}
TEST_F(BrowserAccessibilityAndroidTest,
TestImageInnerText_PendingAdultOrEmpty) {
ui::AXTreeUpdate tree;
tree.root_id = 1;
tree.nodes.resize(5);
tree.nodes[0].id = 1;
tree.nodes[0].child_ids = {2, 3, 4, 5};
tree.nodes[1].id = 2;
tree.nodes[1].role = ax::mojom::Role::kImage;
tree.nodes[1].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationPending);
tree.nodes[2].id = 3;
tree.nodes[2].role = ax::mojom::Role::kImage;
tree.nodes[2].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationEmpty);
tree.nodes[3].id = 4;
tree.nodes[3].role = ax::mojom::Role::kImage;
tree.nodes[3].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationAdult);
tree.nodes[4].id = 5;
tree.nodes[4].role = ax::mojom::Role::kImage;
tree.nodes[4].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed);
std::unique_ptr<BrowserAccessibilityManager> manager(
BrowserAccessibilityManager::Create(
tree, test_browser_accessibility_delegate_.get()));
BrowserAccessibilityAndroid* image_pending =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(0));
BrowserAccessibilityAndroid* image_empty =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(1));
BrowserAccessibilityAndroid* image_adult =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(2));
BrowserAccessibilityAndroid* image_failed =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(3));
EXPECT_EQ(base::ASCIIToUTF16("Getting description..."),
image_pending->GetInnerText());
EXPECT_EQ(base::ASCIIToUTF16("No description available."),
image_empty->GetInnerText());
EXPECT_EQ(base::ASCIIToUTF16(
"Appears to contain adult content. No description available."),
image_adult->GetInnerText());
EXPECT_EQ(base::ASCIIToUTF16("No description available."),
image_failed->GetInnerText());
}
TEST_F(BrowserAccessibilityAndroidTest, TestImageInnerText_Ineligible) {
ui::AXTreeUpdate tree;
tree.root_id = 1;
tree.nodes.resize(5);
tree.nodes[0].id = 1;
tree.nodes[0].child_ids = {2, 3, 4, 5};
tree.nodes[1].id = 2;
tree.nodes[1].role = ax::mojom::Role::kImage;
tree.nodes[1].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kNone);
tree.nodes[2].id = 3;
tree.nodes[2].role = ax::mojom::Role::kImage;
tree.nodes[2].SetName("image_name");
tree.nodes[2].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kWillNotAnnotateDueToScheme);
tree.nodes[3].id = 4;
tree.nodes[3].role = ax::mojom::Role::kImage;
tree.nodes[3].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation);
tree.nodes[4].id = 5;
tree.nodes[4].role = ax::mojom::Role::kImage;
tree.nodes[4].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kSilentlyEligibleForAnnotation);
std::unique_ptr<BrowserAccessibilityManager> manager(
BrowserAccessibilityManager::Create(
tree, test_browser_accessibility_delegate_.get()));
BrowserAccessibilityAndroid* image_none =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(0));
BrowserAccessibilityAndroid* image_scheme =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(1));
BrowserAccessibilityAndroid* image_ineligible =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(2));
BrowserAccessibilityAndroid* image_silent =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(3));
EXPECT_EQ(base::string16(), image_none->GetInnerText());
EXPECT_EQ(base::ASCIIToUTF16("image_name"), image_scheme->GetInnerText());
EXPECT_EQ(base::string16(), image_ineligible->GetInnerText());
EXPECT_EQ(base::string16(), image_silent->GetInnerText());
}
TEST_F(BrowserAccessibilityAndroidTest,
TestImageInnerText_AnnotationSucceeded) {
ui::AXTreeUpdate tree;
tree.root_id = 1;
tree.nodes.resize(3);
tree.nodes[0].id = 1;
tree.nodes[0].child_ids = {2, 3};
tree.nodes[1].id = 2;
tree.nodes[1].role = ax::mojom::Role::kImage;
tree.nodes[1].AddStringAttribute(ax::mojom::StringAttribute::kImageAnnotation,
"test_annotation");
tree.nodes[1].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded);
tree.nodes[2].id = 3;
tree.nodes[2].role = ax::mojom::Role::kImage;
tree.nodes[2].SetName("image_name");
tree.nodes[2].AddStringAttribute(ax::mojom::StringAttribute::kImageAnnotation,
"test_annotation");
tree.nodes[2].SetImageAnnotationStatus(
ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded);
std::unique_ptr<BrowserAccessibilityManager> manager(
BrowserAccessibilityManager::Create(
tree, test_browser_accessibility_delegate_.get()));
BrowserAccessibilityAndroid* image_succeeded =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(0));
BrowserAccessibilityAndroid* image_succeeded_with_name =
static_cast<BrowserAccessibilityAndroid*>(
manager->GetRoot()->PlatformGetChild(1));
EXPECT_EQ(base::ASCIIToUTF16("test_annotation"),
image_succeeded->GetInnerText());
EXPECT_EQ(base::ASCIIToUTF16("image_name, test_annotation"),
image_succeeded_with_name->GetInnerText());
}
} // namespace content
......@@ -841,6 +841,14 @@ below:
</message>
</else>
</if>
<if expr="is_android">
<message name="IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION_ANDROID_LTR" desc="Accessibility message spoken out loud to screen reader users on Android to inform them that they can get a description of an image by opening the 'More Options' menu on the top right. This is only for LTR languages. The 'more options' text should match TC ID 8110367050267853536">
This image isn‘t labeled. Open the More Options menu at the top right to get image descriptions.
</message>
<message name="IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION_ANDROID_RTL" desc="Accessibility message spoken out loud to screen reader users on Android to inform them that they can get a description of an image by opening the 'More Options' menu on the top left. This is only for RTL languages. The 'more options' text should match TC ID 8110367050267853536">
This image isn‘t labeled. Open the More Options menu at the top left to get image descriptions.
</message>
</if>
<message name="IDS_AX_IMAGE_ELIGIBLE_FOR_ANNOTATION" desc="Accessibility message spoken out loud to screen reader users to inform them that they can get a description of an image by opening the context menu.">
To get missing image descriptions, open the context menu.
</message>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment