Commit 918ae51e authored by Clemens Arbesser's avatar Clemens Arbesser Committed by Commit Bot

[Autofill Assistant] Added support for text spans to ui framework.

This comprises style spans like <b> and <i>, as well as text links of
the form <link1>.

Also refactored testing util for style spans.

Bug: b/145043394
Change-Id: I7bcd375db919ad050a4edaace00f2cff1cc61085
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2116233Reviewed-by: default avatarMarian Fechete <marianfe@google.com>
Commit-Queue: Clemens Arbesser <arbesser@google.com>
Cr-Commit-Position: refs/heads/master@{#753566}
parent 5c75624c
......@@ -34,6 +34,12 @@ public class AssistantGenericUiDelegate {
AssistantGenericUiDelegate.this, modelIdentifier, value);
}
void onTextLinkClicked(int link) {
assert mNativeAssistantGenericUiDelegate != 0;
AssistantGenericUiDelegateJni.get().onTextLinkClicked(
mNativeAssistantGenericUiDelegate, AssistantGenericUiDelegate.this, link);
}
@CalledByNative
private void clearNativePtr() {
mNativeAssistantGenericUiDelegate = 0;
......@@ -45,5 +51,7 @@ public class AssistantGenericUiDelegate {
String identifier);
void onValueChanged(long nativeAssistantGenericUiDelegate,
AssistantGenericUiDelegate caller, String modelIdentifier, AssistantValue value);
void onTextLinkClicked(
long nativeAssistantGenericUiDelegate, AssistantGenericUiDelegate caller, int link);
}
}
......@@ -91,11 +91,11 @@ public class AssistantViewFactory {
/** Creates a {@code android.widget.TextView} widget. */
@CalledByNative
public static TextView createTextView(
Context context, String identifier, String text, @Nullable String textAppearance) {
public static TextView createTextView(Context context, AssistantGenericUiDelegate delegate,
String identifier, String text, @Nullable String textAppearance) {
TextView textView = new TextView(context);
AssistantViewInteractions.setViewText(textView, text, delegate);
textView.setTag(identifier);
textView.setText(text);
if (textAppearance != null) {
int styleId = context.getResources().getIdentifier(
textAppearance, "style", context.getPackageName());
......
......@@ -16,6 +16,7 @@ import androidx.annotation.Nullable;
import org.chromium.base.annotations.CalledByNative;
import org.chromium.base.annotations.JNINamespace;
import org.chromium.chrome.browser.autofill.prefeditor.EditorTextField;
import org.chromium.chrome.browser.autofill_assistant.AssistantTextUtils;
import org.chromium.chrome.browser.autofill_assistant.user_data.AssistantDateTime;
import org.chromium.content.browser.input.PopupItemType;
import org.chromium.content.browser.input.SelectPopupDialog;
......@@ -95,12 +96,14 @@ public class AssistantViewInteractions {
}
@CalledByNative
private static boolean setViewText(View view, String text) {
static boolean setViewText(View view, String text, AssistantGenericUiDelegate delegate) {
if (view instanceof TextView) {
((TextView) view).setText(text);
AssistantTextUtils.applyVisualAppearanceTags(
(TextView) view, text, delegate::onTextLinkClicked);
return true;
} else if (view instanceof EditorTextField) {
((EditorTextField) view).getEditText().setText(text);
AssistantTextUtils.applyVisualAppearanceTags(
((EditorTextField) view).getEditText(), text, delegate::onTextLinkClicked);
return true;
}
return false;
......
......@@ -30,10 +30,12 @@ import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.iterableWithSize;
import static org.chromium.chrome.browser.autofill_assistant.AutofillAssistantUiTestUtil.hasTypefaceSpan;
import static org.chromium.chrome.browser.autofill_assistant.AutofillAssistantUiTestUtil.isImportantForAccessibility;
import static org.chromium.chrome.browser.autofill_assistant.AutofillAssistantUiTestUtil.startAutofillAssistant;
import static org.chromium.chrome.browser.autofill_assistant.AutofillAssistantUiTestUtil.waitUntilViewMatchesCondition;
import android.graphics.Typeface;
import android.support.test.InstrumentationRegistry;
import android.support.test.filters.MediumTest;
import android.widget.DatePicker;
......@@ -76,6 +78,7 @@ import org.chromium.chrome.browser.autofill_assistant.proto.InteractionsProto;
import org.chromium.chrome.browser.autofill_assistant.proto.LinearLayoutProto;
import org.chromium.chrome.browser.autofill_assistant.proto.ModelProto;
import org.chromium.chrome.browser.autofill_assistant.proto.OnModelValueChangedEventProto;
import org.chromium.chrome.browser.autofill_assistant.proto.OnTextLinkClickedProto;
import org.chromium.chrome.browser.autofill_assistant.proto.OnUserActionCalled;
import org.chromium.chrome.browser.autofill_assistant.proto.OnViewClickedEventProto;
import org.chromium.chrome.browser.autofill_assistant.proto.ProcessedActionProto;
......@@ -1603,4 +1606,94 @@ public class AutofillAssistantGenericUiTest {
StringList.newBuilder().addValues("test 2")))
.build()));
}
/**
* Tests text spans like <b></b> and <i></i> as well as text links like <link1></link1>.
*/
@Test
@MediumTest
public void testTextSpansAndLinks() {
List<ModelProto.ModelValue> modelValues = new ArrayList<>();
modelValues.add((ModelProto.ModelValue) ModelProto.ModelValue.newBuilder()
.setIdentifier("text_link_clicked")
.build());
List<InteractionProto> interactions = new ArrayList<>();
interactions.add((InteractionProto) InteractionProto.newBuilder()
.setTriggerEvent(EventProto.newBuilder().setOnTextLinkClicked(
OnTextLinkClickedProto.newBuilder().setTextLink(1)))
.addCallbacks(CallbackProto.newBuilder().setSetValue(
SetModelValueProto.newBuilder()
.setValue(ValueProto.newBuilder().setBooleans(
BooleanList.newBuilder().addValues(true)))
.setModelIdentifier("text_link_clicked")))
.addCallbacks(CallbackProto.newBuilder().setEndAction(
EndActionProto.newBuilder().setStatus(
ProcessedActionStatusProto.ACTION_APPLIED)))
.build());
GenericUserInterfaceProto genericUserInterface =
(GenericUserInterfaceProto) GenericUserInterfaceProto.newBuilder()
.setRootView(ViewProto.newBuilder().setViewContainer(
ViewContainerProto.newBuilder()
.setLinearLayout(
LinearLayoutProto.newBuilder().setOrientation(
LinearLayoutProto.Orientation.VERTICAL))
.addViews(ViewProto.newBuilder().setTextView(
TextViewProto.newBuilder().setText(
"<b>bold text</b>")))
.addViews(ViewProto.newBuilder().setTextView(
TextViewProto.newBuilder().setText(
"<i>italic text</i>")))
.addViews(ViewProto.newBuilder().setTextView(
TextViewProto.newBuilder().setText(
"<link1>click here</link1>")))))
.setInteractions(
InteractionsProto.newBuilder().addAllInteractions(interactions))
.setModel(ModelProto.newBuilder().addAllValues(modelValues))
.build();
ArrayList<ActionProto> list = new ArrayList<>();
list.add((ActionProto) ActionProto.newBuilder()
.setShowGenericUi(ShowGenericUiProto.newBuilder()
.setGenericUserInterface(genericUserInterface)
.addOutputModelIdentifiers("text_link_clicked"))
.build());
AutofillAssistantTestScript script = new AutofillAssistantTestScript(
(SupportedScriptProto) SupportedScriptProto.newBuilder()
.setPath("form_target_website.html")
.setPresentation(PresentationProto.newBuilder().setAutostart(true).setChip(
ChipProto.newBuilder().setText("Autostart")))
.build(),
list);
AutofillAssistantTestService testService =
new AutofillAssistantTestService(Collections.singletonList(script));
startAutofillAssistant(mTestRule.getActivity(), testService);
waitUntilViewMatchesCondition(withText("click here"), isCompletelyDisplayed());
onView(withText("bold text"))
.check(matches(hasTypefaceSpan(0, "bold text".length() - 1, Typeface.BOLD)));
onView(withText("italic text"))
.check(matches(hasTypefaceSpan(0, "italic text".length() - 1, Typeface.ITALIC)));
int numNextActionsCalled = testService.getNextActionsCounter();
onView(withText("click here")).perform(click());
testService.waitUntilGetNextActions(numNextActionsCalled + 1);
List<ProcessedActionProto> processedActions = testService.getProcessedActions();
assertThat(processedActions, iterableWithSize(1));
assertThat(
processedActions.get(0).getStatus(), is(ProcessedActionStatusProto.ACTION_APPLIED));
ShowGenericUiProto.Result result = processedActions.get(0).getShowGenericUiResult();
List<ModelProto.ModelValue> resultModelValues = result.getModel().getValuesList();
assertThat(resultModelValues, iterableWithSize(1));
assertThat(resultModelValues.get(0),
is((ModelProto.ModelValue) ModelProto.ModelValue.newBuilder()
.setIdentifier("text_link_clicked")
.setValue(ValueProto.newBuilder().setBooleans(
BooleanList.newBuilder().addValues(true)))
.build()));
}
}
......@@ -10,7 +10,9 @@ import static android.support.test.espresso.matcher.ViewMatchers.assertThat;
import static android.support.test.espresso.matcher.ViewMatchers.withText;
import static org.hamcrest.Matchers.is;
import static org.hamcrest.Matchers.not;
import static org.chromium.chrome.browser.autofill_assistant.AutofillAssistantUiTestUtil.hasTypefaceSpan;
import static org.chromium.chrome.browser.autofill_assistant.AutofillAssistantUiTestUtil.openTextLink;
import static org.chromium.content_public.browser.test.util.TestThreadUtils.runOnUiThreadBlocking;
......@@ -82,26 +84,30 @@ public class AutofillAssistantTextUtilsTest {
TextView boldTextView = runOnUiThreadBlocking(() -> {
TextView view = new TextView(mTestRule.getActivity());
mTestLayout.addView(view);
AssistantTextUtils.applyVisualAppearanceTags(view, "<b>Bold text</b>", null);
StyleSpan[] spans =
((SpannedString) view.getText()).getSpans(0, view.length(), StyleSpan.class);
assertThat(spans.length, is(1));
assertThat(spans[0].getStyle(), is(Typeface.BOLD));
AssistantTextUtils.applyVisualAppearanceTags(
view, "regular text, <b>bold text</b>", null);
return view;
});
onView(is(boldTextView)).check(matches(withText("Bold text")));
int boldStart = "regular text, ".length();
int boldEnd = "regular text, bold text".length() - 1;
onView(withText("regular text, bold text"))
.check(matches(hasTypefaceSpan(boldStart, boldEnd, Typeface.BOLD)));
onView(withText("regular text, bold text"))
.check(matches(not(hasTypefaceSpan(0, boldStart, Typeface.BOLD))));
TextView italicTextView = runOnUiThreadBlocking(() -> {
TextView view = new TextView(mTestRule.getActivity());
mTestLayout.addView(view);
AssistantTextUtils.applyVisualAppearanceTags(view, "<i>Italic text</i>", null);
StyleSpan[] spans =
((SpannedString) view.getText()).getSpans(0, view.length(), StyleSpan.class);
assertThat(spans.length, is(1));
assertThat(spans[0].getStyle(), is(Typeface.ITALIC));
AssistantTextUtils.applyVisualAppearanceTags(
view, "regular text, <i>italic text</i>", null);
return view;
});
onView(is(italicTextView)).check(matches(withText("Italic text")));
int italicStart = "italic text, ".length();
int italicEnd = "italic text, bold text".length() - 1;
onView(withText("regular text, italic text"))
.check(matches(hasTypefaceSpan(italicStart, italicEnd, Typeface.ITALIC)));
onView(withText("regular text, italic text"))
.check(matches(not(hasTypefaceSpan(0, italicStart, Typeface.ITALIC))));
}
@Test
......
......@@ -25,7 +25,9 @@ import android.support.test.espresso.ViewAssertion;
import android.support.test.espresso.core.deps.guava.base.Preconditions;
import android.support.test.espresso.matcher.BoundedMatcher;
import android.text.Spanned;
import android.text.SpannedString;
import android.text.style.ClickableSpan;
import android.text.style.StyleSpan;
import android.util.DisplayMetrics;
import android.view.Gravity;
import android.view.LayoutInflater;
......@@ -127,7 +129,11 @@ class AutofillAssistantUiTestUtil {
};
}
/** Checks that a text view has a specific typeface style. */
/**
* Checks that a text view has a specific typeface style. NOTE: this only works for views that
* explicitly set the text style, *NOT* for text spans! @see {@link #hasTypefaceSpan(int, int,
* int)}
*/
public static TypeSafeMatcher<View> hasTypefaceStyle(/*@Typeface.Style*/ int style) {
return new TypeSafeMatcher<View>() {
@Override
......@@ -149,6 +155,46 @@ class AutofillAssistantUiTestUtil {
};
}
/**
* Checks that a text view has a span with the specified style in the specified region.
* @param start The start offset of the style span
* @param end The end offset of the style span
* @param style The style to check for
* @return A matcher that returns true if the view satisfies the condition.
*/
public static TypeSafeMatcher<View> hasTypefaceSpan(
int start, int end, /*@Typeface.Style*/ int style) {
return new TypeSafeMatcher<View>() {
@Override
protected boolean matchesSafely(View item) {
if (!(item instanceof TextView)) {
return false;
}
TextView textView = (TextView) item;
if (!(textView.getText() instanceof SpannedString)) {
return false;
}
if (start >= textView.length() || end >= textView.length()) {
return false;
}
StyleSpan[] spans =
((SpannedString) textView.getText()).getSpans(start, end, StyleSpan.class);
for (StyleSpan span : spans) {
if (span.getStyle() == style) {
return true;
}
}
return false;
}
@Override
public void describeTo(Description description) {
description.appendText(
"hasTypefaceSpan(" + style + ") in [" + start + ", " + end + "]");
}
};
}
public static Matcher<View> isImportantForAccessibility(int mode) {
return new TypeSafeMatcher<View>() {
@Override
......
......@@ -5,6 +5,7 @@
#include "chrome/browser/android/autofill_assistant/assistant_generic_ui_delegate.h"
#include "base/android/jni_string.h"
#include "base/strings/string_number_conversions.h"
#include "chrome/android/features/autofill_assistant/jni_headers/AssistantGenericUiDelegate_jni.h"
#include "chrome/browser/android/autofill_assistant/ui_controller_android.h"
#include "chrome/browser/android/autofill_assistant/ui_controller_android_utils.h"
......@@ -50,6 +51,14 @@ void AssistantGenericUiDelegate::OnValueChanged(
ui_controller_android_utils::ToNativeValue(env, jvalue));
}
void AssistantGenericUiDelegate::OnTextLinkClicked(
JNIEnv* env,
const base::android::JavaParamRef<jobject>& jcaller,
jint jlink) {
ui_controller_->OnViewEvent(
{EventProto::kOnTextLinkClicked, base::NumberToString(jlink)});
}
base::android::ScopedJavaGlobalRef<jobject>
AssistantGenericUiDelegate::GetJavaObject() {
return java_assistant_generic_ui_delegate_;
......
......@@ -30,6 +30,11 @@ class AssistantGenericUiDelegate {
const base::android::JavaParamRef<jstring>& jmodel_identifier,
const base::android::JavaParamRef<jobject>& jvalue);
// A text link was clicked.
void OnTextLinkClicked(JNIEnv* env,
const base::android::JavaParamRef<jobject>& jcaller,
jint jlink);
base::android::ScopedJavaGlobalRef<jobject> GetJavaObject();
private:
......
......@@ -93,6 +93,7 @@ base::android::ScopedJavaLocalRef<jobject> CreateJavaViewContainer(
base::android::ScopedJavaLocalRef<jobject> CreateJavaTextView(
JNIEnv* env,
const base::android::ScopedJavaLocalRef<jobject>& jcontext,
const base::android::ScopedJavaGlobalRef<jobject>& jdelegate,
const base::android::ScopedJavaLocalRef<jstring>& jidentifier,
const TextViewProto& proto) {
base::android::ScopedJavaLocalRef<jstring> jtext_appearance = nullptr;
......@@ -101,7 +102,7 @@ base::android::ScopedJavaLocalRef<jobject> CreateJavaTextView(
base::android::ConvertUTF8ToJavaString(env, proto.text_appearance());
}
return Java_AssistantViewFactory_createTextView(
env, jcontext, jidentifier,
env, jcontext, jdelegate, jidentifier,
base::android::ConvertUTF8ToJavaString(env, proto.text()),
jtext_appearance);
}
......@@ -129,7 +130,8 @@ base::android::ScopedJavaGlobalRef<jobject> CreateJavaView(
proto.view_container());
break;
case ViewProto::kTextView:
jview = CreateJavaTextView(env, jcontext, jidentifier, proto.text_view());
jview = CreateJavaTextView(env, jcontext, jdelegate, jidentifier,
proto.text_view());
break;
case ViewProto::kDividerView:
jview = Java_AssistantViewFactory_createDividerView(env, jcontext,
......
......@@ -6,6 +6,7 @@
#include "base/android/jni_array.h"
#include "base/android/jni_string.h"
#include "base/optional.h"
#include "base/strings/string_number_conversions.h"
#include "chrome/android/features/autofill_assistant/jni_headers/AssistantViewInteractions_jni.h"
#include "chrome/browser/android/autofill_assistant/ui_controller_android_utils.h"
#include "components/autofill_assistant/browser/user_model.h"
......@@ -210,7 +211,8 @@ void ShowCalendarPopup(base::WeakPtr<UserModel> user_model,
void SetViewText(
base::WeakPtr<UserModel> user_model,
const SetTextProto& proto,
std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views) {
std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views,
base::android::ScopedJavaGlobalRef<jobject> jdelegate) {
if (!user_model) {
return;
}
......@@ -238,7 +240,8 @@ void SetViewText(
JNIEnv* env = base::android::AttachCurrentThread();
Java_AssistantViewInteractions_setViewText(
env, jview->second,
base::android::ConvertUTF8ToJavaString(env, text->strings().values(0)));
base::android::ConvertUTF8ToJavaString(env, text->strings().values(0)),
jdelegate);
}
void SetViewVisibility(
......
......@@ -56,7 +56,8 @@ void ShowCalendarPopup(base::WeakPtr<UserModel> user_model,
void SetViewText(
base::WeakPtr<UserModel> user_model,
const SetTextProto& proto,
std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views);
std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views,
base::android::ScopedJavaGlobalRef<jobject> jdelegate);
// Sets the visibility of a view.
void SetViewVisibility(
......
......@@ -9,6 +9,7 @@
#include "base/callback_helpers.h"
#include "base/memory/weak_ptr.h"
#include "base/optional.h"
#include "base/strings/string_number_conversions.h"
#include "chrome/browser/android/autofill_assistant/generic_ui_controller_android.h"
#include "chrome/browser/android/autofill_assistant/generic_ui_events_android.h"
#include "chrome/browser/android/autofill_assistant/generic_ui_interactions_android.h"
......@@ -46,7 +47,7 @@ base::Optional<EventHandler::EventKey> CreateEventKeyFromProto(
case EventProto::kOnViewClicked: {
auto jview = views->find(proto.on_view_clicked().view_identifier());
if (jview == views->end()) {
VLOG(1) << "Invalid click event, no view with id='"
VLOG(1) << "Invalid OnViewClickedEventProto: no view with id='"
<< proto.on_view_clicked().view_identifier() << "' found";
return base::nullopt;
}
......@@ -55,11 +56,18 @@ base::Optional<EventHandler::EventKey> CreateEventKeyFromProto(
return base::Optional<EventHandler::EventKey>(
{proto.kind_case(), proto.on_view_clicked().view_identifier()});
}
case EventProto::kOnUserActionCalled: {
case EventProto::kOnUserActionCalled:
return base::Optional<EventHandler::EventKey>(
{proto.kind_case(),
proto.on_user_action_called().user_action_identifier()});
}
case EventProto::kOnTextLinkClicked:
if (!proto.on_text_link_clicked().has_text_link()) {
VLOG(1) << "Invalid OnTextLinkClickedProto: no text_link specified";
return base::nullopt;
}
return base::Optional<EventHandler::EventKey>(
{proto.kind_case(),
base::NumberToString(proto.on_text_link_clicked().text_link())});
case EventProto::KIND_NOT_SET:
VLOG(1) << "Error creating event: kind not set";
return base::nullopt;
......@@ -156,8 +164,8 @@ CreateInteractionCallbackFromProto(
}
return base::Optional<InteractionHandlerAndroid::InteractionCallback>(
base::BindRepeating(&android_interactions::SetViewText,
user_model->GetWeakPtr(), proto.set_text(),
views));
user_model->GetWeakPtr(), proto.set_text(), views,
jdelegate));
case CallbackProto::kToggleUserAction:
if (proto.toggle_user_action().user_actions_model_identifier().empty()) {
VLOG(1) << "Error creating ToggleUserAction interaction: "
......
......@@ -40,6 +40,9 @@ std::ostream& operator<<(std::ostream& out,
case EventProto::kOnUserActionCalled:
out << "kOnUserActionCalled";
break;
case EventProto::kOnTextLinkClicked:
out << "kOnTextLinkClicked";
break;
case EventProto::KIND_NOT_SET:
break;
}
......
......@@ -49,6 +49,7 @@ message EventProto {
OnModelValueChangedEventProto on_value_changed = 1;
OnViewClickedEventProto on_view_clicked = 2;
OnUserActionCalled on_user_action_called = 3;
OnTextLinkClickedProto on_text_link_clicked = 4;
}
}
......@@ -70,6 +71,13 @@ message OnUserActionCalled {
optional string user_action_identifier = 1;
}
// Event that is triggered when clicking the specified text link.
message OnTextLinkClickedProto {
// The number of the text link to observe, i.e., the number in the html-style
// tag <link3>...</link3>.
optional int32 text_link = 1;
}
// Callback that writes the specified value to |model_identifier|.
message SetModelValueProto {
// The model identifier to write to.
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment