[Autofill Assistant] Added ToString and SetText interactions.

For dates, a format string must be specified for ToString to work. User
actions are not supported in ToString. SetText only works for TextViews
or subclasses of it.

Bug: b/145043394
Change-Id: Ic44c9dad4ae714afb0d95ccfae00f4f461c546c5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2096439
Commit-Queue: Clemens Arbesser <arbesser@google.com>
Reviewed-by: Sandro Maggi <sandromaggi@google.com>
Cr-Commit-Position: refs/heads/master@{#749207}

chrome/android/features/autofill_assistant/java/src/org/chromium/chrome/browser/autofill_assistant/generic_ui/AssistantViewInteractions.java

@@ -9,6 +9,7 @@ import static org.chromium.chrome.browser.autofill_assistant.generic_ui.Assistan
 import android.content.Context;
 import android.support.annotation.Nullable;
 import android.view.View;
+import android.widget.TextView;
 
 import org.chromium.base.annotations.CalledByNative;
 import org.chromium.base.annotations.JNINamespace;
@@ -93,4 +94,13 @@ public class AssistantViewInteractions {
                 maxDate.getDateTimes().get(0).getTimeInUtcMillis(), -1, null);
         return true;
     }
+
+    @CalledByNative
+    private static boolean setTextViewText(View view, String text) {
+        if (!(view instanceof TextView)) {
+            return false;
+        }
+        ((TextView) view).setText(text);
+        return true;
+    }
 }

chrome/android/features/autofill_assistant/javatests/src/org/chromium/chrome/browser/autofill_assistant/AutofillAssistantGenericUiTest.java

@@ -53,6 +53,7 @@ import org.chromium.chrome.browser.autofill_assistant.proto.CollectUserDataProto
 import org.chromium.chrome.browser.autofill_assistant.proto.CollectUserDataResultProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.ColorProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.ComputeValueProto;
+import org.chromium.chrome.browser.autofill_assistant.proto.DateFormatProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.DateList;
 import org.chromium.chrome.browser.autofill_assistant.proto.DateProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.DividerViewProto;
@@ -74,6 +75,7 @@ import org.chromium.chrome.browser.autofill_assistant.proto.ProcessedActionProto
 import org.chromium.chrome.browser.autofill_assistant.proto.ProcessedActionStatusProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.PromptProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.SetModelValueProto;
+import org.chromium.chrome.browser.autofill_assistant.proto.SetTextProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.SetUserActionsProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.ShapeDrawableProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.ShowCalendarPopupProto;
@@ -84,6 +86,7 @@ import org.chromium.chrome.browser.autofill_assistant.proto.StringList;
 import org.chromium.chrome.browser.autofill_assistant.proto.SupportedScriptProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.SupportedScriptProto.PresentationProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.TextViewProto;
+import org.chromium.chrome.browser.autofill_assistant.proto.ToStringProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.UserActionList;
 import org.chromium.chrome.browser.autofill_assistant.proto.UserActionProto;
 import org.chromium.chrome.browser.autofill_assistant.proto.ValueProto;
@@ -1047,10 +1050,36 @@ public class AutofillAssistantGenericUiTest {
                                                  .setMinDateModelIdentifier("min_date")
                                                  .setMaxDateModelIdentifier("max_date")))
                                  .build());
+        interactions.add(
+                (InteractionProto) InteractionProto.newBuilder()
+                        .setTriggerEvent(EventProto.newBuilder().setOnValueChanged(
+                                OnModelValueChangedEventProto.newBuilder().setModelIdentifier(
+                                        "date")))
+                        .addCallbacks(CallbackProto.newBuilder().setComputeValue(
+                                ComputeValueProto.newBuilder()
+                                        .setResultModelIdentifier("date_string")
+                                        .setToString(
+                                                ToStringProto.newBuilder()
+                                                        .setModelIdentifier("date")
+                                                        .setDateFormat(
+                                                                DateFormatProto.newBuilder()
+                                                                        .setDateFormat(
+                                                                                "EEE, MMM d y")))))
+                        .build());
+        interactions.add(
+                (InteractionProto) InteractionProto.newBuilder()
+                        .setTriggerEvent(EventProto.newBuilder().setOnValueChanged(
+                                OnModelValueChangedEventProto.newBuilder().setModelIdentifier(
+                                        "date_string")))
+                        .addCallbacks(CallbackProto.newBuilder().setSetText(
+                                SetTextProto.newBuilder()
+                                        .setModelIdentifier("date_string")
+                                        .setViewIdentifier("text_view")))
+                        .build());
 
         GenericUserInterfaceProto genericUserInterface =
                 (GenericUserInterfaceProto) GenericUserInterfaceProto.newBuilder()
-                        .setRootView(createTextView("Click me", "text_view"))
+                        .setRootView(createTextView("", "text_view"))
                         .setInteractions(
                                 InteractionsProto.newBuilder().addAllInteractions(interactions))
                         .setModel(ModelProto.newBuilder().addAllValues(modelValues))
@@ -1074,13 +1103,14 @@ public class AutofillAssistantGenericUiTest {
                 new AutofillAssistantTestService(Collections.singletonList(script));
         startAutofillAssistant(mTestRule.getActivity(), testService);
 
-        waitUntilViewMatchesCondition(withText("Done"), isCompletelyDisplayed());
+        waitUntilViewMatchesCondition(withText("Wed, Apr 15, 2020"), isCompletelyDisplayed());
 
-        onView(withText("Click me")).perform(click());
+        onView(withText("Wed, Apr 15, 2020")).perform(click());
         onView(withClassName(equalTo(DatePicker.class.getName())))
                 .inRoot(isDialog())
                 .perform(setDate(2020, 7, 13));
         onView(withText(R.string.date_picker_dialog_set)).inRoot(isDialog()).perform(click());
+        waitUntilViewMatchesCondition(withText("Mon, Jul 13, 2020"), isCompletelyDisplayed());
 
         int numNextActionsCalled = testService.getNextActionsCounter();
         onView(withContentDescription("Done")).perform(click());

chrome/browser/android/autofill_assistant/generic_ui_controller_android.cc

@@ -233,7 +233,7 @@ GenericUiControllerAndroid::CreateFromProto(
   auto interaction_handler =
       std::make_unique<InteractionHandlerAndroid>(event_handler, jcontext);
   if (!interaction_handler->AddInteractionsFromProto(
-          proto.interactions(), env, *views, jdelegate, user_model,
+          proto.interactions(), env, views.get(), jdelegate, user_model,
           basic_interactions)) {
     return nullptr;
   }

chrome/browser/android/autofill_assistant/interaction_handler_android.cc

@@ -206,19 +206,52 @@ void ShowCalendarPopup(base::WeakPtr<UserModel> user_model,
   }
 }
 
+void SetTextViewText(
+    base::WeakPtr<UserModel> user_model,
+    const SetTextProto& proto,
+    std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views) {
+  if (!user_model) {
+    return;
+  }
+
+  auto text = user_model->GetValue(proto.model_identifier());
+  if (!text.has_value()) {
+    DVLOG(2) << "Failed to set text for " << proto.view_identifier() << ": "
+             << proto.model_identifier() << " not found in model";
+    return;
+  }
+  if (text->strings().values_size() != 1) {
+    DVLOG(2) << "Failed to set text for " << proto.view_identifier()
+             << ": expected " << proto.model_identifier()
+             << " to contain single string, but was instead " << *text;
+    return;
+  }
+
+  auto jview = views->find(proto.view_identifier());
+  if (jview == views->end()) {
+    DVLOG(2) << "Failed to set text for " << proto.view_identifier() << ": "
+             << " view not found";
+    return;
+  }
+
+  JNIEnv* env = base::android::AttachCurrentThread();
+  Java_AssistantViewInteractions_setTextViewText(
+      env, jview->second,
+      base::android::ConvertUTF8ToJavaString(env, text->strings().values(0)));
+}
+
 base::Optional<EventHandler::EventKey> CreateEventKeyFromProto(
     const EventProto& proto,
     JNIEnv* env,
-    const std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>&
-        views,
+    std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views,
     base::android::ScopedJavaGlobalRef<jobject> jdelegate) {
   switch (proto.kind_case()) {
     case EventProto::kOnValueChanged:
       return base::Optional<EventHandler::EventKey>(
           {proto.kind_case(), proto.on_value_changed().model_identifier()});
     case EventProto::kOnViewClicked: {
-      auto jview = views.find(proto.on_view_clicked().view_identifier());
-      if (jview == views.end()) {
+      auto jview = views->find(proto.on_view_clicked().view_identifier());
+      if (jview == views->end()) {
         VLOG(1) << "Invalid click event, no view with id='"
                 << proto.on_view_clicked().view_identifier() << "' found";
         return base::nullopt;
@@ -247,6 +280,7 @@ CreateInteractionCallbackFromProto(
     const CallbackProto& proto,
     UserModel* user_model,
     BasicInteractions* basic_interactions,
+    std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views,
     base::android::ScopedJavaGlobalRef<jobject> jcontext,
     base::android::ScopedJavaGlobalRef<jobject> jdelegate) {
   switch (proto.kind_case()) {
@@ -314,7 +348,20 @@ CreateInteractionCallbackFromProto(
           base::BindRepeating(&ShowCalendarPopup, user_model->GetWeakPtr(),
                               proto.show_calendar_popup(), jcontext,
                               jdelegate));
-      break;
+    case CallbackProto::kSetText:
+      if (proto.set_text().model_identifier().empty()) {
+        VLOG(1) << "Error creating SetText interaction: "
+                   "model_identifier not set";
+        return base::nullopt;
+      }
+      if (proto.set_text().view_identifier().empty()) {
+        VLOG(1) << "Error creating SetText interaction: "
+                   "view_identifier not set";
+        return base::nullopt;
+      }
+      return base::Optional<InteractionHandlerAndroid::InteractionCallback>(
+          base::BindRepeating(&SetTextViewText, user_model->GetWeakPtr(),
+                              proto.set_text(), views));
     case CallbackProto::KIND_NOT_SET:
       VLOG(1) << "Error creating interaction: kind not set";
       return base::nullopt;
@@ -348,8 +395,7 @@ void InteractionHandlerAndroid::StopListening() {
 bool InteractionHandlerAndroid::AddInteractionsFromProto(
     const InteractionsProto& proto,
     JNIEnv* env,
-    const std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>&
-        views,
+    std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views,
     base::android::ScopedJavaGlobalRef<jobject> jdelegate,
     UserModel* user_model,
     BasicInteractions* basic_interactions) {
@@ -367,7 +413,8 @@ bool InteractionHandlerAndroid::AddInteractionsFromProto(
 
     for (const auto& callback_proto : interaction_proto.callbacks()) {
       auto callback = CreateInteractionCallbackFromProto(
-          callback_proto, user_model, basic_interactions, jcontext_, jdelegate);
+          callback_proto, user_model, basic_interactions, views, jcontext_,
+          jdelegate);
       if (!callback) {
         VLOG(1) << "Invalid callback for interaction";
         return false;

chrome/browser/android/autofill_assistant/interaction_handler_android.h

@@ -43,11 +43,11 @@ class InteractionHandlerAndroid : public EventHandler::Observer {
 
   // Creates callbacks for each interaction in |proto| as well as the
   // corresponding view events in |views|. Returns false if |proto| is invalid.
+  // |views| must outlive this interaction handler.
   bool AddInteractionsFromProto(
       const InteractionsProto& proto,
       JNIEnv* env,
-      const std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>&
-          views,
+      std::map<std::string, base::android::ScopedJavaGlobalRef<jobject>>* views,
       base::android::ScopedJavaGlobalRef<jobject> jdelegate,
       UserModel* user_model,
       BasicInteractions* basic_interactions);

components/autofill_assistant/browser/basic_interactions.cc

@@ -4,6 +4,9 @@
 
 #include "components/autofill_assistant/browser/basic_interactions.h"
 #include "base/bind_helpers.h"
+#include "base/i18n/time_formatting.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/utf_string_conversions.h"
 #include "components/autofill_assistant/browser/script_executor_delegate.h"
 #include "components/autofill_assistant/browser/trigger_context.h"
 #include "components/autofill_assistant/browser/user_model.h"
@@ -78,6 +81,74 @@ bool BooleanNot(UserModel* user_model,
   return true;
 }
 
+bool ValueToString(UserModel* user_model,
+                   const std::string& result_model_identifier,
+                   const ToStringProto& proto) {
+  auto value = user_model->GetValue(proto.model_identifier());
+  std::string result;
+
+  if (!value.has_value()) {
+    DVLOG(2) << "Error evaluating " << __func__ << ": "
+             << proto.model_identifier() << " not found in model";
+    return false;
+  }
+  if (!AreAllValuesOfSize({*value}, 1)) {
+    DVLOG(2) << "Error evaluating " << __func__
+             << ": expected single value, but got a list instead";
+    return false;
+  }
+  if (AreAllValuesOfType({*value}, ValueProto::kUserActions)) {
+    DVLOG(2) << "Error evaluating " << __func__
+             << ": does not support stringifying user actions";
+    return false;
+  }
+  switch (value->kind_case()) {
+    case ValueProto::kStrings:
+      result = value->strings().values(0);
+      break;
+    case ValueProto::kBooleans:
+      result = value->booleans().values(0) ? "true" : "false";
+      break;
+    case ValueProto::kInts:
+      result = base::NumberToString(value->ints().values(0));
+      break;
+    case ValueProto::kUserActions:
+      NOTREACHED();
+      return false;
+    case ValueProto::kDates: {
+      if (proto.date_format().date_format().empty()) {
+        DVLOG(2) << "Error evaluating " << __func__ << ": date_format not set";
+        return false;
+      }
+      auto date = value->dates().values(0);
+      base::Time::Exploded exploded_time = {date.year(),
+                                            date.month(),
+                                            /* day_of_week = */ -1,
+                                            date.day(),
+                                            /* hour = */ 0,
+                                            /* minute = */ 0,
+                                            /* second = */ 0,
+                                            /* millisecond = */ 0};
+      base::Time time;
+      if (!base::Time::FromLocalExploded(exploded_time, &time)) {
+        DVLOG(2) << "Error evaluating " << __func__ << ": invalid date "
+                 << *value;
+        return false;
+      }
+
+      result = base::UTF16ToUTF8(base::TimeFormatWithPattern(
+          time, proto.date_format().date_format().c_str()));
+      break;
+    }
+    case ValueProto::KIND_NOT_SET:
+      DVLOG(2) << "Error evaluating " << __func__ << ": kind not set";
+      return false;
+  }
+
+  user_model->SetValue(result_model_identifier, SimpleValue(result));
+  return true;
+}
+
 }  // namespace
 
 base::WeakPtr<BasicInteractions> BasicInteractions::GetWeakPtr() {
@@ -129,6 +200,14 @@ bool BasicInteractions::ComputeValue(const ComputeValueProto& proto) {
       }
       return BooleanNot(delegate_->GetUserModel(),
                         proto.result_model_identifier(), proto.boolean_not());
+    case ComputeValueProto::kToString:
+      if (proto.to_string().model_identifier().empty()) {
+        DVLOG(2) << "Error computing ComputeValue::ToString: "
+                    "model_identifier not specified";
+        return false;
+      }
+      return ValueToString(delegate_->GetUserModel(),
+                           proto.result_model_identifier(), proto.to_string());
     case ComputeValueProto::KIND_NOT_SET:
       DVLOG(2) << "Error computing value: kind not set";
       return false;

components/autofill_assistant/browser/basic_interactions_unittest.cc

@@ -4,6 +4,7 @@
 
 #include "components/autofill_assistant/browser/basic_interactions.h"
 #include "base/test/gmock_callback_support.h"
+#include "base/test/icu_test_util.h"
 #include "base/test/mock_callback.h"
 #include "components/autofill_assistant/browser/fake_script_executor_delegate.h"
 #include "components/autofill_assistant/browser/interactions.pb.h"
@@ -12,6 +13,16 @@
 
 namespace autofill_assistant {
 
+namespace {
+DateProto CreateDateProto(int year, int month, int day) {
+  DateProto proto;
+  proto.set_year(year);
+  proto.set_month(month);
+  proto.set_day(day);
+  return proto;
+}
+}  // namespace
+
 class BasicInteractionsTest : public testing::Test {
  protected:
   BasicInteractionsTest() { delegate_.SetUserModel(&user_model_); }
@@ -127,6 +138,64 @@ TEST_F(BasicInteractionsTest, ComputeValueBooleanNot) {
   EXPECT_FALSE(basic_interactions_.ComputeValue(proto));
 }
 
+TEST_F(BasicInteractionsTest, ComputeValueToString) {
+  ComputeValueProto proto;
+  proto.mutable_to_string()->set_model_identifier("value");
+  user_model_.SetValue("value", SimpleValue(1));
+
+  // Result model identifier not set.
+  EXPECT_FALSE(basic_interactions_.ComputeValue(proto));
+
+  // Integer
+  proto.set_result_model_identifier("output");
+  EXPECT_TRUE(basic_interactions_.ComputeValue(proto));
+  EXPECT_EQ(*user_model_.GetValue("output"), SimpleValue(std::string("1")));
+
+  // Boolean
+  user_model_.SetValue("value", SimpleValue(true));
+  EXPECT_TRUE(basic_interactions_.ComputeValue(proto));
+  EXPECT_EQ(*user_model_.GetValue("output"), SimpleValue(std::string("true")));
+
+  // String
+  user_model_.SetValue("value", SimpleValue(std::string("test asd")));
+  EXPECT_TRUE(basic_interactions_.ComputeValue(proto));
+  EXPECT_EQ(*user_model_.GetValue("output"),
+            SimpleValue(std::string("test asd")));
+
+  // Date without format fails.
+  user_model_.SetValue("value", SimpleValue(CreateDateProto(2020, 10, 23)));
+  EXPECT_FALSE(basic_interactions_.ComputeValue(proto));
+
+  // Date with format succeeds.
+  {
+    base::test::ScopedRestoreICUDefaultLocale locale(std::string("en_US"));
+    proto.mutable_to_string()->mutable_date_format()->set_date_format(
+        "EEE, MMM d y");
+    EXPECT_TRUE(basic_interactions_.ComputeValue(proto));
+    EXPECT_EQ(*user_model_.GetValue("output"),
+              SimpleValue(std::string("Fri, Oct 23, 2020")));
+  }
+
+  // Date in german locale.
+  {
+    base::test::ScopedRestoreICUDefaultLocale locale(std::string("de_DE"));
+    EXPECT_TRUE(basic_interactions_.ComputeValue(proto));
+    EXPECT_EQ(*user_model_.GetValue("output"),
+              SimpleValue(std::string("Fr., 23. Okt. 2020")));
+  }
+
+  // Empty value fails.
+  user_model_.SetValue("value", ValueProto());
+  EXPECT_FALSE(basic_interactions_.ComputeValue(proto));
+
+  // Multi value fails.
+  ValueProto multi_value;
+  multi_value.mutable_booleans()->add_values(true);
+  multi_value.mutable_booleans()->add_values(false);
+  user_model_.SetValue("value", multi_value);
+  EXPECT_FALSE(basic_interactions_.ComputeValue(proto));
+}
+
 TEST_F(BasicInteractionsTest, EndActionWithoutCallbackFails) {
   EndActionProto proto;
   ASSERT_DEATH(basic_interactions_.EndAction(proto),

components/autofill_assistant/browser/interactions.proto

@@ -35,6 +35,7 @@ message CallbackProto {
     SetUserActionsProto set_user_actions = 5;
     EndActionProto end_action = 6;
     ShowCalendarPopupProto show_calendar_popup = 7;
+    SetTextProto set_text = 8;
   }
 }
 
@@ -81,6 +82,8 @@ message ComputeValueProto {
     BooleanOrProto boolean_or = 3;
     // Computes the logical NOT of the specified model identifiers.
     BooleanNotProto boolean_not = 4;
+    // Creates a string representation of the specified value.
+    ToStringProto to_string = 5;
   }
 
   // The model identifier to write the result to.
@@ -107,6 +110,21 @@ message BooleanNotProto {
   optional string model_identifier = 1;
 }
 
+// Creates a string representation of the specified value.
+message ToStringProto {
+  // The model identifier to stringify.
+  optional string model_identifier = 1;
+
+  // Optional format options.
+  oneof format_options { DateFormatProto date_format = 2; }
+}
+
+// A format string for a date, such as "EEE, MMM d". See
+// http://userguide.icu-project.org/formatparse/datetime for full specification.
+message DateFormatProto {
+  optional string date_format = 1;
+}
+
 // Displays a standard info popup.
 message ShowInfoPopupProto {
   optional InfoPopupProto info_popup = 1;
@@ -169,3 +187,12 @@ message ShowCalendarPopupProto {
   // DateList.
   optional string max_date_model_identifier = 3;
 }
+
+// Modifies the displayed text of a text view.
+message SetTextProto {
+  // The model identifier containing the text to set. Must point to a single
+  // string.
+  optional string model_identifier = 1;
+  // The text view to modify. Must point to a text view.
+  optional string view_identifier = 2;
+}