Commit 83d6e446 authored by Donn Denman's avatar Donn Denman Committed by Commit Bot

[TTS] Add new Blink tap-signals for ML.

Adds new feature signals from Blink about the page at the
location tapped.  These signals will be useful for deciding
whether to trigger Contextual Search.

Signals include the text run-length of the element tapped and
the effective font size.

BUG=754862

Change-Id: I02227479ecfae4d3b174a1845fed7c3d24e355b8
Reviewed-on: https://chromium-review.googlesource.com/920325Reviewed-by: default avatarDaniel Cheng <dcheng@chromium.org>
Commit-Queue: Donn Denman <donnd@chromium.org>
Cr-Commit-Position: refs/heads/master@{#543713}
parent f144f1eb
...@@ -4382,19 +4382,27 @@ class MockUnhandledTapNotifierImpl : public mojom::blink::UnhandledTapNotifier { ...@@ -4382,19 +4382,27 @@ class MockUnhandledTapNotifierImpl : public mojom::blink::UnhandledTapNotifier {
mojom::blink::UnhandledTapInfoPtr unhandled_tap_info) override { mojom::blink::UnhandledTapInfoPtr unhandled_tap_info) override {
was_unhandled_tap_ = true; was_unhandled_tap_ = true;
tapped_position_ = unhandled_tap_info->tapped_position_in_viewport; tapped_position_ = unhandled_tap_info->tapped_position_in_viewport;
element_text_run_length_ = unhandled_tap_info->element_text_run_length;
font_size_ = unhandled_tap_info->font_size_in_pixels;
} }
bool WasUnhandledTap() const { return was_unhandled_tap_; } bool WasUnhandledTap() const { return was_unhandled_tap_; }
int GetTappedXPos() const { return tapped_position_.X(); } int GetTappedXPos() const { return tapped_position_.X(); }
int GetTappedYPos() const { return tapped_position_.Y(); } int GetTappedYPos() const { return tapped_position_.Y(); }
int GetFontSize() const { return font_size_; }
int GetElementTextRunLength() const { return element_text_run_length_; }
void Reset() { void Reset() {
was_unhandled_tap_ = false; was_unhandled_tap_ = false;
tapped_position_ = IntPoint(); tapped_position_ = IntPoint();
element_text_run_length_ = 0;
font_size_ = 0;
binding_.Close(); binding_.Close();
} }
private: private:
bool was_unhandled_tap_ = false; bool was_unhandled_tap_ = false;
IntPoint tapped_position_; IntPoint tapped_position_;
int element_text_run_length_ = 0;
int font_size_ = 0;
mojo::Binding<mojom::blink::UnhandledTapNotifier> binding_; mojo::Binding<mojom::blink::UnhandledTapNotifier> binding_;
}; };
...@@ -4468,6 +4476,8 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeeded) { ...@@ -4468,6 +4476,8 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeeded) {
EXPECT_TRUE(mock_notifier_.WasUnhandledTap()); EXPECT_TRUE(mock_notifier_.WasUnhandledTap());
EXPECT_EQ(64, mock_notifier_.GetTappedXPos()); EXPECT_EQ(64, mock_notifier_.GetTappedXPos());
EXPECT_EQ(278, mock_notifier_.GetTappedYPos()); EXPECT_EQ(278, mock_notifier_.GetTappedYPos());
EXPECT_EQ(16, mock_notifier_.GetFontSize());
EXPECT_EQ(7, mock_notifier_.GetElementTextRunLength());
// Test basic tap handling and notification. // Test basic tap handling and notification.
Tap("target"); Tap("target");
...@@ -4483,6 +4493,8 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeeded) { ...@@ -4483,6 +4493,8 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeeded) {
EXPECT_TRUE(mock_notifier_.WasUnhandledTap()); EXPECT_TRUE(mock_notifier_.WasUnhandledTap());
EXPECT_EQ(188, mock_notifier_.GetTappedXPos()); EXPECT_EQ(188, mock_notifier_.GetTappedXPos());
EXPECT_EQ(124, mock_notifier_.GetTappedYPos()); EXPECT_EQ(124, mock_notifier_.GetTappedYPos());
EXPECT_EQ(16, mock_notifier_.GetFontSize());
EXPECT_EQ(28, mock_notifier_.GetElementTextRunLength());
} }
TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithMutateDom) { TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithMutateDom) {
...@@ -4514,7 +4526,7 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithPreventDefault) { ...@@ -4514,7 +4526,7 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithPreventDefault) {
} }
TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithNonTriggeringNodes) { TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithNonTriggeringNodes) {
Tap("checkbox"); Tap("image");
EXPECT_FALSE(mock_notifier_.WasUnhandledTap()); EXPECT_FALSE(mock_notifier_.WasUnhandledTap());
Tap("editable"); Tap("editable");
...@@ -4524,6 +4536,16 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithNonTriggeringNodes) { ...@@ -4524,6 +4536,16 @@ TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithNonTriggeringNodes) {
EXPECT_FALSE(mock_notifier_.WasUnhandledTap()); EXPECT_FALSE(mock_notifier_.WasUnhandledTap());
} }
TEST_P(ShowUnhandledTapTest, ShowUnhandledTapUIIfNeededWithTextSizes) {
Tap("large");
EXPECT_TRUE(mock_notifier_.WasUnhandledTap());
EXPECT_EQ(20, mock_notifier_.GetFontSize());
Tap("small");
EXPECT_TRUE(mock_notifier_.WasUnhandledTap());
EXPECT_EQ(10, mock_notifier_.GetFontSize());
}
#endif // BUILDFLAG(ENABLE_UNHANDLED_TAP) #endif // BUILDFLAG(ENABLE_UNHANDLED_TAP)
TEST_P(WebViewTest, StopLoadingIfJavaScriptURLReturnsNoStringResult) { TEST_P(WebViewTest, StopLoadingIfJavaScriptURLReturnsNoStringResult) {
......
...@@ -21,7 +21,11 @@ ...@@ -21,7 +21,11 @@
#include "public/public_features.h" #include "public/public_features.h"
#if BUILDFLAG(ENABLE_UNHANDLED_TAP) #if BUILDFLAG(ENABLE_UNHANDLED_TAP)
#include "core/editing/FrameSelection.h"
#include "core/editing/SelectionTemplate.h"
#include "core/editing/VisibleSelection.h"
#include "core/frame/LocalFrameClient.h" #include "core/frame/LocalFrameClient.h"
#include "core/style/ComputedStyle.h"
#include "public/platform/unhandled_tap_notifier.mojom-blink.h" #include "public/platform/unhandled_tap_notifier.mojom-blink.h"
#include "public/web/WebNode.h" #include "public/web/WebNode.h"
#include "services/service_manager/public/cpp/interface_provider.h" #include "services/service_manager/public/cpp/interface_provider.h"
...@@ -310,7 +314,7 @@ WebInputEventResult GestureManager::HandleGestureTap( ...@@ -310,7 +314,7 @@ WebInputEventResult GestureManager::HandleGestureTap(
frame_->GetPage()->GetVisualViewport().RootFrameToViewport( frame_->GetPage()->GetVisualViewport().RootFrameToViewport(
tapped_position); tapped_position);
ShowUnhandledTapUIIfNeeded(dom_tree_changed, style_changed, tapped_node, ShowUnhandledTapUIIfNeeded(dom_tree_changed, style_changed, tapped_node,
tapped_position_in_viewport); tapped_element, tapped_position_in_viewport);
} }
return event_result; return event_result;
} }
...@@ -447,9 +451,11 @@ void GestureManager::ShowUnhandledTapUIIfNeeded( ...@@ -447,9 +451,11 @@ void GestureManager::ShowUnhandledTapUIIfNeeded(
bool dom_tree_changed, bool dom_tree_changed,
bool style_changed, bool style_changed,
Node* tapped_node, Node* tapped_node,
Element* tapped_element,
const IntPoint& tapped_position_in_viewport) { const IntPoint& tapped_position_in_viewport) {
#if BUILDFLAG(ENABLE_UNHANDLED_TAP) #if BUILDFLAG(ENABLE_UNHANDLED_TAP)
WebNode web_node(tapped_node); WebNode web_node(tapped_node);
// TODO(donnd): roll in ML-identified signals for suppression once identified.
bool should_trigger = !dom_tree_changed && !style_changed && bool should_trigger = !dom_tree_changed && !style_changed &&
tapped_node->IsTextNode() && tapped_node->IsTextNode() &&
!web_node.IsContentEditable() && !web_node.IsContentEditable() &&
...@@ -457,12 +463,29 @@ void GestureManager::ShowUnhandledTapUIIfNeeded( ...@@ -457,12 +463,29 @@ void GestureManager::ShowUnhandledTapUIIfNeeded(
// Renderer-side trigger-filtering to minimize messaging. // Renderer-side trigger-filtering to minimize messaging.
// The Browser may do additional trigger-filtering. // The Browser may do additional trigger-filtering.
if (should_trigger) { if (should_trigger) {
WebPoint point(tapped_position_in_viewport.X(), // Start setting up the Mojo interface connection.
tapped_position_in_viewport.Y());
auto tapped_info = mojom::blink::UnhandledTapInfo::New(point);
mojom::blink::UnhandledTapNotifierPtr provider; mojom::blink::UnhandledTapNotifierPtr provider;
frame_->Client()->GetInterfaceProvider()->GetInterface( frame_->Client()->GetInterfaceProvider()->GetInterface(
mojo::MakeRequest(&provider)); mojo::MakeRequest(&provider));
// Extract text run-length.
int text_run_length = 0;
if (tapped_element)
text_run_length = tapped_element->textContent().length();
// Compute the style of the tapped node to extract text characteristics.
const ComputedStyle* style = tapped_node->EnsureComputedStyle();
int font_size = 0;
if (style)
font_size = style->FontSize();
// TODO(donnd): get the text color and style and return,
// e.g. style->GetFontWeight() to return bold. Need italic, color, etc.
// Notify the Browser.
WebPoint point(tapped_position_in_viewport.X(),
tapped_position_in_viewport.Y());
auto tapped_info =
mojom::blink::UnhandledTapInfo::New(point, font_size, text_run_length);
provider->ShowUnhandledTapUIIfNeeded(std::move(tapped_info)); provider->ShowUnhandledTapUIIfNeeded(std::move(tapped_info));
} }
#endif // BUILDFLAG(ENABLE_UNHANDLED_TAP) #endif // BUILDFLAG(ENABLE_UNHANDLED_TAP)
......
...@@ -64,6 +64,7 @@ class CORE_EXPORT GestureManager ...@@ -64,6 +64,7 @@ class CORE_EXPORT GestureManager
void ShowUnhandledTapUIIfNeeded(bool dom_tree_changed, void ShowUnhandledTapUIIfNeeded(bool dom_tree_changed,
bool style_changed, bool style_changed,
Node* tapped_node, Node* tapped_node,
Element* tapped_element,
const IntPoint& tapped_position_in_viewport); const IntPoint& tapped_position_in_viewport);
// NOTE: If adding a new field to this class please ensure that it is // NOTE: If adding a new field to this class please ensure that it is
......
...@@ -16,16 +16,25 @@ ...@@ -16,16 +16,25 @@
:indeterminate + span { :indeterminate + span {
background-color: blue; background-color: blue;
} }
#large {
font-size: 20px;
}
#small {
font-size: 10px;
}
</style> </style>
</head> </head>
<body> <body>
<div style="position: absolute; left: 8px; top: 66px; width: 400px; height: 30px;"> <div style="position: absolute; left: 8px; top: 66px; width: 400px; height: 350px;">
<span id="target">This has multiple listeners.</span> <span id="target">This has multiple listeners.</span>
<div id="mutate">This block gets mutated to change the DOM.</div> <div id="mutate">This block gets mutated to change the DOM.</div>
<div id="style_active">This block gets red if active</div> <div id="style_active">This block gets red if active</div>
<input id="checkbox" type="checkbox"><span>indeterminate checkbox</span> <input id="checkbox" type="checkbox"><span>indeterminate checkbox</span><br>
<input id="editable" type="text" value="editable"> <img id="image" width="10" height="10" src="data:image/gif;base64,R0lGODlhAQABAIAAAAUEBAAAACwAAAAAAQABAAACAkQBADs="><br>
<div id="focusable" tabindex="1">focusable</div> <span id="editable" contenteditable="true">editable</span><br>
<span id="focusable" tabindex="1">focusable</span><br>
<span id="large">large</span><br>
<span id="small">small</span><br>
</div> </div>
<p style="position: absolute; left: 8px; top: 400px; width: 400px; height: 30px;"><span id="bottom">Bottom.</span></p> <p style="position: absolute; left: 8px; top: 400px; width: 400px; height: 30px;"><span id="bottom">Bottom.</span></p>
...@@ -33,14 +42,17 @@ ...@@ -33,14 +42,17 @@
<script> <script>
var test; var test;
var mutations = 0; var mutations = 0;
// Remembers which test to run.
function setTest(whichTest) { function setTest(whichTest) {
test = whichTest; test = whichTest;
} }
// Determines if we currently have a test for the given event/handler pair.
function hasTest(operation, handler) { function hasTest(operation, handler) {
var candidate = operation + '-' + handler; var candidate = operation + '-' + handler;
var result = test === candidate; var result = test === candidate;
return result; return result;
} }
// Our universal event handler. Changes the page based on the stored operation.
function handle(event) { function handle(event) {
var operation = event.type; var operation = event.type;
if (hasTest(operation, 'mutateDom')) { if (hasTest(operation, 'mutateDom')) {
...@@ -54,6 +66,7 @@ function handle(event) { ...@@ -54,6 +66,7 @@ function handle(event) {
event.preventDefault(); event.preventDefault();
} }
} }
// Set up $target to have the handle() handler for all tap-related events.
var t = document.getElementById('target'); var t = document.getElementById('target');
t.addEventListener('mousedown', handle); t.addEventListener('mousedown', handle);
t.addEventListener('mouseup', handle); t.addEventListener('mouseup', handle);
......
...@@ -11,6 +11,9 @@ import "ui/gfx/geometry/mojo/geometry.mojom"; ...@@ -11,6 +11,9 @@ import "ui/gfx/geometry/mojo/geometry.mojom";
// This data is used to notifying the Browser to show UI for an unhandled tap if needed. // This data is used to notifying the Browser to show UI for an unhandled tap if needed.
struct UnhandledTapInfo { struct UnhandledTapInfo {
gfx.mojom.Point tapped_position_in_viewport; gfx.mojom.Point tapped_position_in_viewport;
// These values default to 0, which indicates an undetermined value.
uint32 font_size_in_pixels = 0;
uint32 element_text_run_length = 0;
}; };
interface UnhandledTapNotifier { interface UnhandledTapNotifier {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment