Skip to content

GitLab

T
tangled
Commit 3a5c8f4c authored by Michael Martis's avatar Michael Martis Committed by Commit Bot
Browse files
Options

Image annotation service: request and expose image descriptions. 

This change updates the HTTP request sent to the image annotation
server to request image descriptions as well as OCR, and updates
the service API to expose this data.

This change has been manually tested, but more unittests are to
come shortly in another CL.

Bug: 916420

Change-Id: Ifd919c376f117e25db93389ab4091c18c684e016
Reviewed-on: https://chromium-review.googlesource.com/c/1478652
Commit-Queue: Michael Martis <martis@chromium.org>
Reviewed-by: default avatarDominic Mazzoni <dmazzoni@chromium.org>
Reviewed-by: default avatarSam McNally <sammc@chromium.org>
Reviewed-by: default avatarMartin Šrámek <msramek@chromium.org>
Reviewed-by: default avatarAndrew Moylan <amoylan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#635423}
parent 440f9a32
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 266 additions and 121 deletions
components/page_image_annotation/core/page_annotator_unittest.cc
View file @ 3a5c8f4c
......@@ -181,8 +181,13 @@ TEST(PageAnnotatorTest, Annotation) {
// Expect success and failure to be reported.
const auto error = ia_mojom::AnnotateImageResult::NewErrorCode(
ia_mojom::AnnotateImageError::kCanceled);
// Can't use an initializer list since it performs copies.
std::vector<ia_mojom::AnnotationPtr> annotations;
annotations.push_back(ia_mojom::Annotation::New(
ia_mojom::AnnotationType::kOcr, 1.0, "text from image"));
const auto success =
ia_mojom::AnnotateImageResult::NewOcrText("text from image");
ia_mojom::AnnotateImageResult::NewAnnotations(std::move(annotations));
ASSERT_THAT(test_annotator.callbacks_, SizeIs(3));
std::move(test_annotator.callbacks_[0]).Run(error.Clone());
......
content/renderer/accessibility/ax_image_annotator.cc
View file @ 3a5c8f4c
......@@ -177,21 +177,31 @@ void AXImageAnnotator::OnImageAnnotated(
// TODO(nektar): Set the image annotation status on this image to Error.
if (result->is_error_code())
return;
if (!result->is_ocr_text()) {
DLOG(WARNING) << "Unrecognized image annotation result.";
if (!result->is_annotations()) {
DLOG(WARNING) << "No image annotation results.";
return;
}
if (result->get_ocr_text().empty())
return;
for (const auto& annotation : result->get_annotations()) {
if (annotation->type != image_annotation::mojom::AnnotationType::kOcr)
continue;
if (annotation->text.empty())
return;
auto contextualized_string = GetContentClient()->GetLocalizedString(
IDS_AX_IMAGE_ANNOTATION_OCR_CONTEXT,
base::UTF8ToUTF16(annotation->text));
auto contextualized_string = GetContentClient()->GetLocalizedString(
IDS_AX_IMAGE_ANNOTATION_OCR_CONTEXT,
base::UTF8ToUTF16(result->get_ocr_text()));
image_annotations_.at(image.AxID())
.set_annotation(base::UTF16ToUTF8(contextualized_string));
render_accessibility_->MarkWebAXObjectDirty(image, false /* subtree */);
return;
}
image_annotations_.at(image.AxID())
.set_annotation(base::UTF16ToUTF8(contextualized_string));
render_accessibility_->MarkWebAXObjectDirty(image, false /* subtree */);
DLOG(WARNING) << "No OCR results.";
}
} // namespace content
services/image_annotation/annotator.cc
View file @ 3a5c8f4c
This diff is collapsed.
services/image_annotation/annotator.h
View file @ 3a5c8f4c
......@@ -88,14 +88,14 @@ class Annotator : public mojom::Annotator {
using HttpRequestQueue =
std::deque<std::pair<std::string, std::vector<uint8_t>>>;
// Constructs and returns a JSON object containing an OCR request for the
// Constructs and returns a JSON object containing an request for the
// given images.
static std::string FormatJsonOcrRequest(HttpRequestQueue::iterator begin_it,
HttpRequestQueue::iterator end_it);
static std::string FormatJsonRequest(HttpRequestQueue::iterator begin_it,
HttpRequestQueue::iterator end_it);
// Creates a URL loader that calls the image annotation server with an OCR
// request for the given images.
static std::unique_ptr<network::SimpleURLLoader> MakeOcrRequestLoader(
// Creates a URL loader that calls the image annotation server with an
// annotation request for the given images.
static std::unique_ptr<network::SimpleURLLoader> MakeRequestLoader(
const GURL& server_url,
const std::string& api_key,
HttpRequestQueue::iterator begin_it,
......@@ -123,9 +123,9 @@ class Annotator : public mojom::Annotator {
UrlLoaderList::iterator http_request_it,
std::unique_ptr<std::string> json_response);
// Maps from source ID to previously-obtained OCR result.
// Maps from source ID to previously-obtained annotation results.
// TODO(crbug.com/916420): periodically clear entries from this cache.
std::map<std::string, std::string> cached_results_;
std::map<std::string, mojom::AnnotateImageResultPtr> cached_results_;
// Maps from source ID to the list of request info (i.e. info of clients that
// have made requests) for that source.
......
services/image_annotation/annotator_unittest.cc
View file @ 3a5c8f4c
......@@ -30,7 +30,7 @@ using testing::Eq;
using testing::IsEmpty;
using testing::SizeIs;
constexpr char kTestServerUrl[] = "https://ia-pa.googleapis.com/v1/ocr";
constexpr char kTestServerUrl[] = "https://ia-pa.googleapis.com/v1/annotation";
// Example image URLs.
......@@ -46,9 +46,10 @@ constexpr char kTemplateRequest[] = R"(
"imageRequests": [{
"imageId": "%s",
"imageBytes": "%s",
"engineParameters": [{
"ocrParameters": {}
}]
"engineParameters": [
{"ocrParameters": {}},
{"descriptionParameters": {}}
]
}]
}
)";
......@@ -60,23 +61,26 @@ constexpr char kBatchRequest[] = R"(
{
"imageId": "https://www.example.com/image1.jpg",
"imageBytes": "AQID",
"engineParameters": [{
"ocrParameters": {}
}]
"engineParameters": [
{"ocrParameters": {}},
{"descriptionParameters": {}}
]
},
{
"imageId": "https://www.example.com/image2.jpg",
"imageBytes": "BAUG",
"engineParameters": [{
"ocrParameters": {}
}]
"engineParameters": [
{"ocrParameters": {}},
{"descriptionParameters": {}}
]
},
{
"imageId": "https://www.example.com/image3.jpg",
"imageBytes": "BwgJ",
"engineParameters": [{
"ocrParameters": {}
}]
"engineParameters": [
{"ocrParameters": {}},
{"descriptionParameters": {}}
]
}
]
})";
......@@ -317,13 +321,15 @@ void ReportResult(base::Optional<mojom::AnnotateImageError>* const error,
if (result->which() == mojom::AnnotateImageResult::Tag::ERROR_CODE) {
*error = result->get_error_code();
} else {
*ocr_text = std::move(result->get_ocr_text());
CHECK_EQ(result->get_annotations().size(), 1u);
CHECK_EQ(result->get_annotations()[0]->type, mojom::AnnotationType::kOcr);
*ocr_text = std::move(result->get_annotations()[0]->text);
}
}
} // namespace
// Test that OCR works for one client, and that the cache is populated.
// Test that annotation works for one client, and that the cache is populated.
TEST(AnnotatorTest, SuccessAndCache) {
base::test::ScopedTaskEnvironment test_task_env(
base::test::ScopedTaskEnvironment::MainThreadType::MOCK_TIME);
......@@ -361,7 +367,7 @@ TEST(AnnotatorTest, SuccessAndCache) {
// HTTP request should have been made.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -424,7 +430,7 @@ TEST(AnnotatorTest, HttpError) {
// HTTP request should have been made.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
"", net::HTTP_INTERNAL_SERVER_ERROR);
test_task_env.RunUntilIdle();
......@@ -469,7 +475,7 @@ TEST(AnnotatorTest, BackendError) {
// HTTP request should have been made.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kErrorResponse, net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -515,7 +521,7 @@ TEST(AnnotatorTest, ServerError) {
// HTTP request should have been made; respond with nonsense string.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
"Hello, world!", net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -576,7 +582,7 @@ TEST(AnnotatorTest, ProcessorFails) {
// HTTP request for image 1 should have been made.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -638,7 +644,7 @@ TEST(AnnotatorTest, ProcessorDies) {
// HTTP request for image 1 should have been made.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -701,7 +707,7 @@ TEST(AnnotatorTest, ConcurrentSameBatch) {
// A single HTTP request for all images should have been sent.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */, ReformatJson(kBatchRequest),
"annotation", {} /* expected_headers */, ReformatJson(kBatchRequest),
kBatchResponse, net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -767,7 +773,7 @@ TEST(AnnotatorTest, ConcurrentSeparateBatches) {
// still waiting to make the batch that will include the request for image
// 2).
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
R"({
"results": [{
......@@ -791,7 +797,7 @@ TEST(AnnotatorTest, ConcurrentSeparateBatches) {
// Now the HTTP request for image 2 should have been made.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage2Url, "BAUG")),
R"({
"results": [{
......@@ -896,7 +902,7 @@ TEST(AnnotatorTest, DuplicateWork) {
// HTTP request for the image should have been made (with bytes obtained from
// processor 1).
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {} /* expected_headers */,
"annotation", {} /* expected_headers */,
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
test_task_env.RunUntilIdle();
......@@ -940,7 +946,7 @@ TEST(AnnotatorTest, ApiKey) {
// HTTP request should have been made with the API key included.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {{Annotator::kGoogApiKeyHeader, "my_api_key"}},
"annotation", {{Annotator::kGoogApiKeyHeader, "my_api_key"}},
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
}
......@@ -952,7 +958,7 @@ TEST(AnnotatorTest, ApiKey) {
TestServerURLLoaderFactory test_url_factory(
"http://ia-pa.googleapis.com/v1/");
Annotator annotator(GURL("http://ia-pa.googleapis.com/v1/ocr"),
Annotator annotator(GURL("http://ia-pa.googleapis.com/v1/annotation"),
"my_api_key", kThrottle, 1 /* batch_size */,
1.0 /* min_ocr_confidence */,
test_url_factory.AsSharedURLLoaderFactory());
......@@ -972,7 +978,7 @@ TEST(AnnotatorTest, ApiKey) {
// HTTP request should have been made without the API key included.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {{Annotator::kGoogApiKeyHeader, base::nullopt}},
"annotation", {{Annotator::kGoogApiKeyHeader, base::nullopt}},
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
}
......@@ -981,8 +987,8 @@ TEST(AnnotatorTest, ApiKey) {
{
TestServerURLLoaderFactory test_url_factory("https://datascraper.com/");
Annotator annotator(GURL("https://datascraper.com/ocr"), "my_api_key",
kThrottle, 1 /* batch_size */,
Annotator annotator(GURL("https://datascraper.com/annotation"),
"my_api_key", kThrottle, 1 /* batch_size */,
1.0 /* min_ocr_confidence */,
test_url_factory.AsSharedURLLoaderFactory());
TestImageProcessor processor;
......@@ -1001,10 +1007,12 @@ TEST(AnnotatorTest, ApiKey) {
// HTTP request should have been made without the API key included.
test_url_factory.ExpectRequestAndSimulateResponse(
"ocr", {{Annotator::kGoogApiKeyHeader, base::nullopt}},
"annotation", {{Annotator::kGoogApiKeyHeader, base::nullopt}},
ReformatJson(base::StringPrintf(kTemplateRequest, kImage1Url, "AQID")),
kSuccessResponse, net::HTTP_OK);
}
}
// TODO(crbug.com/916420): add unit tests for description annotations.
} // namespace image_annotation
services/image_annotation/public/mojom/image_annotation.mojom
View file @ 3a5c8f4c
......@@ -12,30 +12,48 @@ interface ImageProcessor {
GetJpgImageData() => (array<uint8> bytes);
};
// The ways in which an annotation request can fail.
enum AnnotateImageError {
kCanceled,
kFailure,
kAdult,
};
// The types of annotations that can be returned.
enum AnnotationType {
kOcr,
kLabel,
kCaption,
};
// One annotation for an image.
struct Annotation {
AnnotationType type;
double score;
string text;
};
union AnnotateImageResult {
AnnotateImageError error_code;
string ocr_text;
// If the union is of this type, |annotations| will be non-empty.
array<Annotation> annotations;
};
interface Annotator {
// Requests a11y annotations (i.e. OCR) for the given image.
// Requests a11y annotations (i.e. OCR, labels) for the given image.
//
// |source_id| is either the URL for an image, or some non-URL string that
// uniquely identifies an image (e.g. a hash of image content for a data
// URI). Source IDs are used to query local and remote caches.
//
// |result| will contain either the |error_code| value specifying how
// annotation failed, or the text |ocr_text| extracted from the image (with
// an empty string denoting no image text).
// |result| will contain either the error code specifying how annotation
// failed, or the annotations extracted from the image.
//
// TODO(crbug.com/916420): expand this signature to include a request
// argument when we support more than one type of
// annotation.
// TODO(crbug.com/916420): add a language code as an arg / parameter.
AnnotateImage(string source_id, ImageProcessor image_processor)
=> (AnnotateImageResult result);
};
tools/traffic_annotation/summary/annotations.xml
View file @ 3a5c8f4c
......@@ -138,7 +138,7 @@ Refer to README.md for content description and update process.
<item id="https_server_previews_navigation" hash_code="35725390" type="0" content_hash_code="84423109" os_list="linux,windows" file_path="chrome/browser/previews/previews_lite_page_serving_url_loader.cc"/>
<item id="icon_cacher" hash_code="103133150" type="0" content_hash_code="116368348" os_list="linux,windows" file_path="components/ntp_tiles/icon_cacher_impl.cc"/>
<item id="icon_catcher_get_large_icon" hash_code="44494884" type="0" content_hash_code="98262037" os_list="linux,windows" file_path="components/ntp_tiles/icon_cacher_impl.cc"/>
<item id="image_annotation" hash_code="107881858" type="0" content_hash_code="121106764" os_list="linux,windows" file_path="services/image_annotation/annotator.cc"/>
<item id="image_annotation" hash_code="107881858" type="0" content_hash_code="96203979" os_list="linux,windows" file_path="services/image_annotation/annotator.cc"/>
<item id="indexed_db_internals_handler" hash_code="131180348" type="0" content_hash_code="59026406" os_list="linux,windows" file_path="content/browser/indexed_db/indexed_db_internals_ui.cc"/>
<item id="interest_feed_send" hash_code="76717919" type="0" content_hash_code="34678180" os_list="linux,windows" file_path="components/feed/core/feed_networking_host.cc"/>
<item id="intranet_redirect_detector" hash_code="21785164" type="0" content_hash_code="62025595" os_list="linux,windows" file_path="chrome/browser/intranet_redirect_detector.cc"/>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment