Sets the image annotation status when the service request has failed, cancelled, or is adult

Also ensures that an empty annotation will set the appropriate status

Bug: 905419
TBR: nasko@chromium.org
Change-Id: I09eb4ddc6489f015ff137564c89cbc88328e80cc
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1506373
Commit-Queue: Nektarios Paisios <nektar@chromium.org>
Auto-Submit: Nektarios Paisios <nektar@chromium.org>
Reviewed-by: Dominic Mazzoni <dmazzoni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#638377}

content/renderer/accessibility/ax_image_annotator.cc

@@ -49,6 +49,15 @@ std::string AXImageAnnotator::GetImageAnnotation(
   return std::string();
 }
 
+ax::mojom::ImageAnnotationStatus AXImageAnnotator::GetImageAnnotationStatus(
+    blink::WebAXObject& image) const {
+  DCHECK(!image.IsDetached());
+  const auto lookup = image_annotations_.find(image.AxID());
+  if (lookup != image_annotations_.end())
+    return lookup->second.status();
+  return ax::mojom::ImageAnnotationStatus::kNone;
+}
+
 bool AXImageAnnotator::HasAnnotationInCache(blink::WebAXObject& image) const {
   DCHECK(!image.IsDetached());
   if (!HasImageInCache(image))
@@ -157,6 +166,7 @@ void AXImageAnnotator::MarkAllImagesDirty() {
 AXImageAnnotator::ImageInfo::ImageInfo(const blink::WebAXObject& image)
     : image_processor_(
           base::BindRepeating(&AXImageAnnotator::GetImageData, image)),
+      status_(ax::mojom::ImageAnnotationStatus::kAnnotationPending),
       annotation_(base::nullopt) {}
 
 AXImageAnnotator::ImageInfo::~ImageInfo() = default;
@@ -167,7 +177,24 @@ AXImageAnnotator::ImageInfo::GetImageProcessor() {
 }
 
 bool AXImageAnnotator::ImageInfo::HasAnnotation() const {
-  return annotation_.has_value();
+  switch (status_) {
+    case ax::mojom::ImageAnnotationStatus::kNone:
+    case ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation:
+    // The user hasn't requested an annotation yet, or a previously pending
+    // annotation request had been cancelled.
+    case ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation:
+    case ax::mojom::ImageAnnotationStatus::kAnnotationPending:
+      return false;
+    case ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded:
+      DCHECK(annotation_.has_value());
+      return true;
+    case ax::mojom::ImageAnnotationStatus::kAnnotationEmpty:
+    // Image has been classified as adult content.
+    case ax::mojom::ImageAnnotationStatus::kAnnotationAdult:
+    case ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed:
+      DCHECK(!annotation_.has_value());
+      return true;
+  }
 }
 
 // static
@@ -186,16 +213,43 @@ SkBitmap AXImageAnnotator::GetImageData(const blink::WebAXObject& image) {
 void AXImageAnnotator::OnImageAnnotated(
     const blink::WebAXObject& image,
     image_annotation::mojom::AnnotateImageResultPtr result) {
-  if (image.IsDetached())
-    return;
   if (!base::ContainsKey(image_annotations_, image.AxID()))
     return;
-  // TODO(nektar): Set the image annotation status on this image to Error.
-  if (result->is_error_code())
+
+  if (image.IsDetached()) {
+    image_annotations_.at(image.AxID())
+        .set_status(ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation);
+    // We should not mark dirty a detached object.
     return;
+  }
+
+  if (result->is_error_code()) {
+    DLOG(WARNING) << "Image annotation error.";
+    switch (result->get_error_code()) {
+      case image_annotation::mojom::AnnotateImageError::kCanceled:
+        image_annotations_.at(image.AxID())
+            .set_status(
+                ax::mojom::ImageAnnotationStatus::kEligibleForAnnotation);
+        break;
+      case image_annotation::mojom::AnnotateImageError::kFailure:
+        image_annotations_.at(image.AxID())
+            .set_status(
+                ax::mojom::ImageAnnotationStatus::kAnnotationProcessFailed);
+        break;
+      case image_annotation::mojom::AnnotateImageError::kAdult:
+        image_annotations_.at(image.AxID())
+            .set_status(ax::mojom::ImageAnnotationStatus::kAnnotationAdult);
+        break;
+    }
+    render_accessibility_->MarkWebAXObjectDirty(image, false /* subtree */);
+    return;
+  }
 
   if (!result->is_annotations()) {
     DLOG(WARNING) << "No image annotation results.";
+    image_annotations_.at(image.AxID())
+        .set_status(ax::mojom::ImageAnnotationStatus::kAnnotationEmpty);
+    render_accessibility_->MarkWebAXObjectDirty(image, false /* subtree */);
     return;
   }
 
@@ -229,9 +283,15 @@ void AXImageAnnotator::OnImageAnnotated(
     }
   }
 
-  if (contextualized_strings.empty())
+  if (contextualized_strings.empty()) {
+    image_annotations_.at(image.AxID())
+        .set_status(ax::mojom::ImageAnnotationStatus::kAnnotationEmpty);
+    render_accessibility_->MarkWebAXObjectDirty(image, false /* subtree */);
     return;
+  }
 
+  image_annotations_.at(image.AxID())
+      .set_status(ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded);
   // TODO(accessibility): join two sentences together in a more i18n-friendly
   // way. Since this is intended for a screen reader, though, a period
   // probably works in almost all languages.

content/renderer/accessibility/ax_image_annotator.h

@@ -18,6 +18,7 @@
 #include "services/image_annotation/public/cpp/image_processor.h"
 #include "services/image_annotation/public/mojom/image_annotation.mojom.h"
 #include "third_party/skia/include/core/SkBitmap.h"
+#include "ui/accessibility/ax_enums.mojom.h"
 
 namespace blink {
 
@@ -42,6 +43,8 @@ class CONTENT_EXPORT AXImageAnnotator : public base::CheckedObserver {
   void Destroy();
 
   std::string GetImageAnnotation(blink::WebAXObject& image) const;
+  ax::mojom::ImageAnnotationStatus GetImageAnnotationStatus(
+      blink::WebAXObject& image) const;
   bool HasAnnotationInCache(blink::WebAXObject& image) const;
   bool HasImageInCache(const blink::WebAXObject& image) const;
 
@@ -59,6 +62,13 @@ class CONTENT_EXPORT AXImageAnnotator : public base::CheckedObserver {
     image_annotation::mojom::ImageProcessorPtr GetImageProcessor();
     bool HasAnnotation() const;
 
+    ax::mojom::ImageAnnotationStatus status() const { return status_; }
+
+    void set_status(ax::mojom::ImageAnnotationStatus status) {
+      DCHECK_NE(status, ax::mojom::ImageAnnotationStatus::kNone);
+      status_ = status;
+    }
+
     std::string annotation() const {
       DCHECK(annotation_.has_value());
       return annotation_.value_or("");
@@ -68,6 +78,7 @@ class CONTENT_EXPORT AXImageAnnotator : public base::CheckedObserver {
 
    private:
     image_annotation::ImageProcessor image_processor_;
+    ax::mojom::ImageAnnotationStatus status_;
     base::Optional<std::string> annotation_;
   };
 

content/renderer/accessibility/blink_ax_tree_source.cc

@@ -1062,6 +1062,8 @@ void BlinkAXTreeSource::AddImageAnnotations(blink::WebAXObject src,
   // unloaded images where the size is unknown.
   if (dst->relative_bounds.bounds.width() < kMinImageAnnotationWidth ||
       dst->relative_bounds.bounds.height() < kMinImageAnnotationHeight) {
+    dst->SetImageAnnotationStatus(
+        ax::mojom::ImageAnnotationStatus::kIneligibleForAnnotation);
     return;
   }
 
@@ -1075,7 +1077,7 @@ void BlinkAXTreeSource::AddImageAnnotations(blink::WebAXObject src,
     dst->AddStringAttribute(ax::mojom::StringAttribute::kImageAnnotation,
                             image_annotator_->GetImageAnnotation(src));
     dst->SetImageAnnotationStatus(
-        ax::mojom::ImageAnnotationStatus::kAnnotationSucceeded);
+        image_annotator_->GetImageAnnotationStatus(src));
   } else if (image_annotator_->HasImageInCache(src)) {
     image_annotator_->OnImageUpdated(src);
     dst->SetImageAnnotationStatus(

ui/accessibility/ax_enums.mojom

@@ -558,6 +558,9 @@ enum IntAttribute {
 
   kHasPopup,
 
+  // Image annotation status, of type ImageAnnotationStatus.
+  kImageAnnotationStatus,
+
    // Indicates if a form control has invalid input or
    // if an element has an aria-invalid attribute.
   kInvalidState,
@@ -577,9 +580,6 @@ enum IntAttribute {
    // Focus traversal in views and Android.
   kPreviousFocusId,
   kNextFocusId,
-
-   // Image annotation status, of type ImageAnnotationStatus.
-  kImageAnnotationStatus,
 };
 
 enum FloatAttribute {