Autofill: Support the server predictions

Make the server prediction available as ViewType, separate it from
mServerType to consistent with AutofillServer protobuf.

Add the tests to cover it.

Bug: 1151542
Change-Id: I1083cdc621d586d32327ee40e973b26f3d724f8f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2623673
Commit-Queue: Michael Bai <michaelbai@chromium.org>
Reviewed-by: Robert Sesek <rsesek@chromium.org>
Reviewed-by: Dominic Battré <battre@chromium.org>
Cr-Commit-Position: refs/heads/master@{#843640}

components/autofill/android/provider/form_data_android.cc

@@ -101,10 +101,15 @@ void FormDataAndroid::UpdateFieldTypes(const FormStructure& form_structure) {
   auto form_field_data_android = fields_.begin();
   for (const auto& autofill_field : form_structure) {
     DCHECK(form_field_data_android->get()->SimilarFieldAs(*autofill_field));
+    std::vector<AutofillType> server_predictions;
+    for (const auto& prediction : autofill_field->server_predictions()) {
+      server_predictions.emplace_back(
+          static_cast<ServerFieldType>(prediction.type()));
+    }
     form_field_data_android->get()->UpdateAutofillTypes(
         AutofillType(autofill_field->heuristic_type()),
         AutofillType(autofill_field->server_type()),
-        autofill_field->ComputedType());
+        autofill_field->ComputedType(), server_predictions);
     if (++form_field_data_android == fields_.end())
       break;
   }

components/autofill/android/provider/form_field_data_android.cc

@@ -21,9 +21,28 @@ using base::android::ToJavaArrayOfStrings;
 
 namespace autofill {
 
+namespace {
+base::android::ScopedJavaLocalRef<jobjectArray> ToJavaArrayOfPredictionString(
+    JNIEnv* env,
+    const std::vector<AutofillType>& server_predictions) {
+  if (!server_predictions.empty()) {
+    std::vector<std::string> server_prediction_array;
+    server_prediction_array.reserve(server_predictions.size());
+    for (const auto& p : server_predictions) {
+      server_prediction_array.emplace_back(p.ToString());
+    }
+    return ToJavaArrayOfStrings(env, server_prediction_array);
+  }
+  return nullptr;
+}
+
+}  // namespace
+
 FormFieldDataAndroid::FormFieldDataAndroid(FormFieldData* field)
     : heuristic_type_(AutofillType(UNKNOWN_TYPE)), field_ptr_(field) {}
 
+FormFieldDataAndroid::~FormFieldDataAndroid() = default;
+
 ScopedJavaLocalRef<jobject> FormFieldDataAndroid::GetJavaPeer() {
   JNIEnv* env = AttachCurrentThread();
   ScopedJavaLocalRef<jobject> obj = java_ref_.get(env);
@@ -51,10 +70,12 @@ ScopedJavaLocalRef<jobject> FormFieldDataAndroid::GetJavaPeer() {
       jheuristic_type =
           ConvertUTF8ToJavaString(env, heuristic_type_.ToString());
     }
-    ScopedJavaLocalRef<jstring> jserver_type;
-    jserver_type = ConvertUTF8ToJavaString(env, server_type_.ToString());
-    ScopedJavaLocalRef<jstring> jcomputed_type;
-    jcomputed_type = ConvertUTF8ToJavaString(env, computed_type_.ToString());
+    ScopedJavaLocalRef<jstring> jserver_type =
+        ConvertUTF8ToJavaString(env, server_type_.ToString());
+    ScopedJavaLocalRef<jstring> jcomputed_type =
+        ConvertUTF8ToJavaString(env, computed_type_.ToString());
+    ScopedJavaLocalRef<jobjectArray> jserver_predictions =
+        ToJavaArrayOfPredictionString(env, server_predictions_);
 
     ScopedJavaLocalRef<jobjectArray> jdatalist_values =
         ToJavaArrayOfStrings(env, field_ptr_->datalist_values);
@@ -66,10 +87,10 @@ ScopedJavaLocalRef<jobject> FormFieldDataAndroid::GetJavaPeer() {
         field_ptr_->should_autocomplete, jplaceholder, jtype, jid,
         joption_values, joption_contents, IsCheckable(field_ptr_->check_status),
         IsChecked(field_ptr_->check_status), field_ptr_->max_length,
-        jheuristic_type, jserver_type, jcomputed_type, field_ptr_->bounds.x(),
-        field_ptr_->bounds.y(), field_ptr_->bounds.right(),
-        field_ptr_->bounds.bottom(), jdatalist_values, jdatalist_labels,
-        field_ptr_->IsVisible());
+        jheuristic_type, jserver_type, jcomputed_type, jserver_predictions,
+        field_ptr_->bounds.x(), field_ptr_->bounds.y(),
+        field_ptr_->bounds.right(), field_ptr_->bounds.bottom(),
+        jdatalist_values, jdatalist_labels, field_ptr_->IsVisible());
     java_ref_ = JavaObjectWeakGlobalRef(env, obj);
   }
   return obj;
@@ -113,10 +134,12 @@ bool FormFieldDataAndroid::SimilarFieldAs(const FormFieldData& field) const {
 void FormFieldDataAndroid::UpdateAutofillTypes(
     const AutofillType& heuristic_type,
     const AutofillType& server_type,
-    const AutofillType& computed_type) {
+    const AutofillType& computed_type,
+    const std::vector<AutofillType>& server_predictions) {
   heuristic_type_ = heuristic_type;
   server_type_ = server_type;
   computed_type_ = computed_type;
+  server_predictions_ = server_predictions;
 
   // Java peer isn't available when this object is instantiated, update to
   // Java peer if the prediction arrives later.
@@ -125,12 +148,15 @@ void FormFieldDataAndroid::UpdateAutofillTypes(
   if (obj.is_null())
     return;
 
-  ScopedJavaLocalRef<jstring> jserver_type;
-  jserver_type = ConvertUTF8ToJavaString(env, server_type_.ToString());
-  ScopedJavaLocalRef<jstring> jcomputed_type;
-  jcomputed_type = ConvertUTF8ToJavaString(env, computed_type_.ToString());
+  ScopedJavaLocalRef<jstring> jserver_type =
+      ConvertUTF8ToJavaString(env, server_type_.ToString());
+  ScopedJavaLocalRef<jstring> jcomputed_type =
+      ConvertUTF8ToJavaString(env, computed_type_.ToString());
+  ScopedJavaLocalRef<jobjectArray> jserver_predictions =
+      ToJavaArrayOfPredictionString(env, server_predictions_);
 
-  Java_FormFieldData_updateFieldTypes(env, obj, jserver_type, jcomputed_type);
+  Java_FormFieldData_updateFieldTypes(env, obj, jserver_type, jcomputed_type,
+                                      jserver_predictions);
 }
 
 }  // namespace autofill

components/autofill/android/provider/form_field_data_android.h

@@ -16,8 +16,8 @@ namespace autofill {
 // autofill::FormFieldData available in Java.
 class FormFieldDataAndroid {
  public:
-  FormFieldDataAndroid(FormFieldData* field);
-  virtual ~FormFieldDataAndroid() {}
+  explicit FormFieldDataAndroid(FormFieldData* field);
+  virtual ~FormFieldDataAndroid();
 
   base::android::ScopedJavaLocalRef<jobject> GetJavaPeer();
   void GetValue();
@@ -25,12 +25,14 @@ class FormFieldDataAndroid {
   bool SimilarFieldAs(const FormFieldData& field) const;
   void UpdateAutofillTypes(const AutofillType& heuristic_type,
                            const AutofillType& server_type,
-                           const AutofillType& computed_type);
+                           const AutofillType& computed_type,
+                           const std::vector<AutofillType>& server_predictions);
 
  private:
   AutofillType heuristic_type_;
   AutofillType server_type_;
   AutofillType computed_type_;
+  std::vector<AutofillType> server_predictions_;
 
   // Not owned.
   FormFieldData* field_ptr_;

components/autofill/android/provider/java/src/org/chromium/components/autofill/AutofillProvider.java

@@ -138,8 +138,13 @@ public class AutofillProvider {
                 if (isQueryServerFieldTypesEnabled()) {
                     builder.addAttribute("crowdsourcing-autofill-hints", field.getServerType());
                     builder.addAttribute("computed-autofill-hints", field.getComputedType());
+                    // Compose multiple predictions to a string separated by ','.
+                    String[] predictions = field.getServerPredictions();
+                    if (predictions != null && predictions.length > 0) {
+                        builder.addAttribute("crowdsourcing-predictions-autofill-hints",
+                                String.join(",", predictions));
+                    }
                 }
-
                 switch (field.getControlType()) {
                     case FormFieldData.ControlType.LIST:
                         child.setAutofillType(View.AUTOFILL_TYPE_LIST);
@@ -281,8 +286,8 @@ public class AutofillProvider {
             if (success) {
                 ArrayList<ViewType> viewTypes = new ArrayList<ViewType>();
                 for (FormFieldData field : mFormData.mFields) {
-                    viewTypes.add(new ViewType(
-                            field.getAutofillId(), field.getServerType(), field.getComputedType()));
+                    viewTypes.add(new ViewType(field.getAutofillId(), field.getServerType(),
+                            field.getComputedType(), field.getServerPredictions()));
                 }
                 mAutofillHintsService.onViewTypeAvailable(viewTypes);
             } else {

components/autofill/android/provider/java/src/org/chromium/components/autofill/FormFieldData.java

@@ -68,14 +68,15 @@ public class FormFieldData {
     // after the object instantiated.
     private String mServerType;
     private String mComputedType;
+    private String[] mServerPredictions;
     private AutofillId mAutofillId;
 
     private FormFieldData(String name, String label, String value, String autocompleteAttr,
             boolean shouldAutocomplete, String placeholder, String type, String id,
             String[] optionValues, String[] optionContents, boolean isCheckField, boolean isChecked,
-            int maxLength, String heuristicType, String serverType, String computedType, float left,
-            float top, float right, float bottom, String[] datalistValues, String[] datalistLabels,
-            boolean visible) {
+            int maxLength, String heuristicType, String serverType, String computedType,
+            String[] serverPredictions, float left, float top, float right, float bottom,
+            String[] datalistValues, String[] datalistLabels, boolean visible) {
         mName = name;
         mLabel = label;
         mValue = value;
@@ -101,6 +102,7 @@ public class FormFieldData {
         mMaxLength = maxLength;
         mHeuristicType = heuristicType;
         mServerType = serverType;
+        mServerPredictions = serverPredictions;
         mComputedType = computedType;
         mBounds = new RectF(left, top, right, bottom);
         mVisible = visible;
@@ -147,9 +149,11 @@ public class FormFieldData {
     }
 
     @CalledByNative
-    private void updateFieldTypes(String serverType, String computedType) {
+    private void updateFieldTypes(
+            String serverType, String computedType, String[] serverPredictions) {
         mServerType = serverType;
         mComputedType = computedType;
+        mServerPredictions = serverPredictions;
     }
 
     public String getServerType() {
@@ -160,6 +164,10 @@ public class FormFieldData {
         return mComputedType;
     }
 
+    public String[] getServerPredictions() {
+        return mServerPredictions;
+    }
+
     @CalledByNative
     public boolean isChecked() {
         return mIsChecked;
@@ -188,11 +196,11 @@ public class FormFieldData {
             String autocompleteAttr, boolean shouldAutocomplete, String placeholder, String type,
             String id, String[] optionValues, String[] optionContents, boolean isCheckField,
             boolean isChecked, int maxLength, String heuristicType, String serverType,
-            String computedType, float left, float top, float right, float bottom,
-            String[] datalistValues, String[] datalistLabels, boolean visible) {
+            String computedType, String[] serverPredictions, float left, float top, float right,
+            float bottom, String[] datalistValues, String[] datalistLabels, boolean visible) {
         return new FormFieldData(name, label, value, autocompleteAttr, shouldAutocomplete,
                 placeholder, type, id, optionValues, optionContents, isCheckField, isChecked,
-                maxLength, heuristicType, serverType, computedType, left, top, right, bottom,
-                datalistValues, datalistLabels, visible);
+                maxLength, heuristicType, serverType, computedType, serverPredictions, left, top,
+                right, bottom, datalistValues, datalistLabels, visible);
     }
 }

components/autofill/android/provider/java/src/org/chromium/components/autofill_public/README.md

@@ -43,7 +43,11 @@
     ```java
     public void onViewTypeAvailable(List<ViewType> viewTypeList) {
         for(ViewType viewType : viewTypeList) {
-            Log.d("MyAutofillService", viewType.mAutofillId.toString() + ":" + viewType.mType);
+          if (viewType.getServerPredictions() ! = null) {
+              // Uses server predictions if they are available.
+          } else {
+              // otherwise, uses viewType.mServerType.
+          }
         }
     }
     ```

components/autofill/android/provider/java/src/org/chromium/components/autofill_public/ViewType.java

@@ -14,6 +14,9 @@ import org.chromium.base.annotations.VerifiesOnO;
 
 /**
  * This class is used to send the server and computed view type to the autofill service.
+ * The valid types are listed in the two FieldTypeToStringPiece() functions in
+ * components/autofill/core/browser/field_types.cc. Note that the list of possibly returned strings
+ * can and will change in the future.
  */
 @TargetApi(Build.VERSION_CODES.O)
 @VerifiesOnO
@@ -24,17 +27,17 @@ public class ViewType implements Parcelable {
     public final AutofillId mAutofillId;
 
     /**
-     * The type from Chrome autofill server. The valid types are listed in the two
-     * FieldTypeToStringPiece() functions in components/autofill/core/browser/field_types.cc. Note
-     * that the list of possibly returned strings can and will change in the future.
+     * The type from Chrome autofill server.
      */
     public final String mServerType;
 
     /**
-     * The type computed overall type. The valid types types are the same as for mServerType.
+     * The type computed overall type. The valid types are the same as for mServerType.
      */
     public final String mComputedType;
 
+    private String[] mServerPredictions;
+
     public static final Parcelable.Creator<ViewType> CREATOR = new Parcelable.Creator<ViewType>() {
         @Override
         public ViewType createFromParcel(Parcel in) {
@@ -47,17 +50,21 @@ public class ViewType implements Parcelable {
         }
     };
 
-    public ViewType(AutofillId id, String serverType, String computedType) {
+    public ViewType(
+            AutofillId id, String serverType, String computedType, String[] serverPredictions) {
         mAutofillId = id;
         mServerType = serverType;
         mComputedType = computedType;
+        mServerPredictions = serverPredictions;
     }
 
     private ViewType(Parcel in) {
         mAutofillId = AutofillId.CREATOR.createFromParcel(in);
         mServerType = in.readString();
         mComputedType = in.readString();
+        in.readStringArray(mServerPredictions);
     }
+
     @Override
     public int describeContents() {
         return 0;
@@ -68,5 +75,14 @@ public class ViewType implements Parcelable {
         mAutofillId.writeToParcel(parcel, flags);
         parcel.writeString(mServerType);
         parcel.writeString(mComputedType);
+        parcel.writeStringArray(mServerPredictions);
+    }
+
+    /**
+     * @return the server predictions, they are in the order of the confidence. The mServerType
+     * shall be used if the server predictions aren't available.
+     */
+    public String[] getServerPredictions() {
+        return mServerPredictions;
     }
 }

components/autofill/android/provider/junit/src/org/chromium/components/autofill/AutofillProviderTest.java

@@ -83,11 +83,11 @@ public class AutofillProviderTest {
     public void testTransformFormFieldToContainViewCoordinates() {
         ArrayList<FormFieldData> fields = new ArrayList<FormFieldData>(1);
         fields.add(FormFieldData.createFormFieldData(null, null, null, null, false, null, null,
-                null, null, null, false, false, 0, null, null, null, 10 /* left */, 20 /* top */,
-                300 /* right */, 60 /*bottom*/, null, null, true));
+                null, null, null, false, false, 0, null, null, null, null, 10 /* left */,
+                20 /* top */, 300 /* right */, 60 /*bottom*/, null, null, true));
         fields.add(FormFieldData.createFormFieldData(null, null, null, null, false, null, null,
-                null, null, null, false, false, 0, null, null, null, 20 /* left */, 100 /* top */,
-                400 /* right */, 200 /*bottom*/, null, null, true));
+                null, null, null, false, false, 0, null, null, null, null, 20 /* left */,
+                100 /* top */, 400 /* right */, 200 /*bottom*/, null, null, true));
         FormData formData = new FormData(null, null, fields);
         mAutofillProvider.transformFormFieldToContainViewCoordinates(formData);
         RectF result = formData.mFields.get(0).getBoundsInContainerViewCoordinates();

components/autofill/android/provider/test_support/autofill_provider_test_helper.cc

@@ -85,6 +85,58 @@ JNI_AutofillProviderTestHelper_SimulateMainFrameAutofillServerResponseForTesting
   return true;
 }
 
+static jboolean
+JNI_AutofillProviderTestHelper_SimulateMainFramePredictionsAutofillServerResponseForTesting(
+    JNIEnv* env,
+    const base::android::JavaParamRef<jobject>& jweb_contents,
+    const base::android::JavaParamRef<jobjectArray>& jfield_ids,
+    const base::android::JavaParamRef<jobjectArray>& jfield_types) {
+  std::vector<base::string16> field_ids;
+  base::android::AppendJavaStringArrayToStringVector(env, jfield_ids,
+                                                     &field_ids);
+  std::vector<std::vector<int>> field_types;
+  base::android::JavaArrayOfIntArrayToIntVector(env, jfield_types,
+                                                &field_types);
+
+  AutofillHandler* autofill_handler = ToMainFrameAutofillHandler(jweb_contents);
+  const std::map<FormRendererId, std::unique_ptr<FormStructure>>&
+      form_structures = autofill_handler->form_structures();
+  CHECK(!form_structures.empty());
+
+  // Make API response with suggestions.
+  AutofillQueryResponse response;
+  AutofillQueryResponse::FormSuggestion* form_suggestion;
+
+  form_suggestion = response.add_form_suggestions();
+  size_t found_fields_count = 0;
+  std::vector<FormSignature> signatures;
+  for (auto& j : form_structures) {
+    FormData formData = j.second->ToFormData();
+    for (size_t i = 0; i < field_ids.size(); ++i) {
+      for (auto form_field_data : formData.fields) {
+        if (form_field_data.id_attribute == field_ids[i]) {
+          autofill::test::AddFieldPredictionsToForm(
+              form_field_data, field_types[i], form_suggestion);
+          found_fields_count++;
+          break;
+        }
+      }
+    }
+    if (found_fields_count > 0) {
+      signatures = autofill::test::GetEncodedSignatures(*(j.second));
+      CHECK(found_fields_count == field_ids.size());
+    }
+  }
+
+  std::string response_string;
+  CHECK(response.SerializeToString(&response_string));
+  std::string encoded_response_string;
+  base::Base64Encode(response_string, &encoded_response_string);
+  autofill_handler->OnLoadedServerPredictionsForTest(encoded_response_string,
+                                                     signatures);
+  return true;
+}
+
 static void
 JNI_AutofillProviderTestHelper_SimulateMainFrameAutofillQueryFailedForTesting(
     JNIEnv* env,

components/autofill/android/provider/test_support/java/src/org/chromium/components/autofill/AutofillProviderTestHelper.java

@@ -19,6 +19,9 @@ import org.chromium.content_public.browser.WebContents;
 @TargetApi(Build.VERSION_CODES.O)
 @JNINamespace("autofill")
 public class AutofillProviderTestHelper {
+    /**
+     * Simulate the primary server type only.
+     */
     public static boolean simulateMainFrameAutofillServerResponseForTesting(
             WebContents webContents, String[] fieldIds, int[] fieldTypes) {
         return AutofillProviderTestHelperJni.get()
@@ -26,6 +29,16 @@ public class AutofillProviderTestHelper {
                         webContents, fieldIds, fieldTypes);
     }
 
+    /**
+     * Simulate the server predictions, the first prediction will be set as primary server type.
+     */
+    public static boolean simulateMainFramePredictionsAutofillServerResponseForTesting(
+            WebContents webContents, String[] fieldIds, int[][] fieldTypes) {
+        return AutofillProviderTestHelperJni.get()
+                .simulateMainFramePredictionsAutofillServerResponseForTesting(
+                        webContents, fieldIds, fieldTypes);
+    }
+
     public static void simulateMainFrameAutofillQueryFailedForTesting(WebContents webContents) {
         AutofillProviderTestHelperJni.get().simulateMainFrameAutofillQueryFailedForTesting(
                 webContents);
@@ -35,6 +48,8 @@ public class AutofillProviderTestHelper {
     interface Natives {
         boolean simulateMainFrameAutofillServerResponseForTesting(
                 WebContents webContents, String[] fieldIds, int[] fieldTypes);
+        boolean simulateMainFramePredictionsAutofillServerResponseForTesting(
+                WebContents webContents, String[] fieldIds, int[][] fieldTypes);
         void simulateMainFrameAutofillQueryFailedForTesting(WebContents webContents);
     }
 }

components/autofill/core/browser/autofill_test_utils.cc

@@ -909,7 +909,7 @@ std::vector<FormSignature> GetEncodedSignatures(
 }
 
 void AddFieldSuggestionToForm(
-    autofill::FormFieldData field_data,
+    const autofill::FormFieldData& field_data,
     ServerFieldType field_type,
     ::autofill::AutofillQueryResponse_FormSuggestion* form_suggestion) {
   auto* field_suggestion = form_suggestion->add_field_suggestions();
@@ -918,5 +918,32 @@ void AddFieldSuggestionToForm(
   field_suggestion->set_primary_type_prediction(field_type);
 }
 
+void AddFieldPredictionsToForm(
+    const autofill::FormFieldData& field_data,
+    const std::vector<int>& field_types,
+    ::autofill::AutofillQueryResponse_FormSuggestion* form_suggestion) {
+  std::vector<ServerFieldType> types;
+  for (auto type : field_types) {
+    types.emplace_back(static_cast<ServerFieldType>(type));
+  }
+  AddFieldPredictionsToForm(field_data, types, form_suggestion);
+}
+
+void AddFieldPredictionsToForm(
+    const autofill::FormFieldData& field_data,
+    const std::vector<ServerFieldType>& field_types,
+    ::autofill::AutofillQueryResponse_FormSuggestion* form_suggestion) {
+  // According to api_v1.proto, the first element is always set to primary type.
+  auto* field_suggestion = form_suggestion->add_field_suggestions();
+  field_suggestion->set_field_signature(
+      CalculateFieldSignatureForField(field_data).value());
+  field_suggestion->set_primary_type_prediction(*field_types.begin());
+  for (auto field_type : field_types) {
+    AutofillQueryResponse_FormSuggestion_FieldSuggestion_FieldPrediction*
+        prediction = field_suggestion->add_predictions();
+    prediction->set_type(field_type);
+  }
+}
+
 }  // namespace test
 }  // namespace autofill

components/autofill/core/browser/autofill_test_utils.h

@@ -338,10 +338,22 @@ std::string NextYear();
 std::string TenYearsFromNow();
 
 void AddFieldSuggestionToForm(
-    autofill::FormFieldData field_data,
+    const autofill::FormFieldData& field_data,
     ServerFieldType field_type,
     ::autofill::AutofillQueryResponse_FormSuggestion* form_suggestion);
 
+// Adds |field_types| predictions of |field_data| to |form_suggestion| query
+// response. Assumes int type can be cast to ServerFieldType.
+void AddFieldPredictionsToForm(
+    const autofill::FormFieldData& field_data,
+    const std::vector<int>& field_types,
+    ::autofill::AutofillQueryResponse_FormSuggestion* form_suggestion);
+
+void AddFieldPredictionsToForm(
+    const autofill::FormFieldData& field_data,
+    const std::vector<ServerFieldType>& field_types,
+    ::autofill::AutofillQueryResponse_FormSuggestion* form_suggestion);
+
 }  // namespace test
 }  // namespace autofill