[autofill] Make form_structure_browsertest.cc use embedded_test_server.

The data driven tests in form_structure_browsertes.cc were navigating to
test pages using data URLs. These have no way to generate relative URLs
for form actions, so the output was being generated incorrectly for
forms that would be evaluated as !ShouldBeParsed due to having "search"
actions.

This CL uses the embedded test server to serve the test pages instead.

Bug: 789944
Change-Id: I9bdb6fa376dba070195d125807fb5e4b5443a203
Reviewed-on: https://chromium-review.googlesource.com/804054
Commit-Queue: Roger McFarlane <rogerm@chromium.org>
Reviewed-by: Mathieu Perreault <mathp@chromium.org>
Cr-Commit-Position: refs/heads/master@{#521517}

chrome/browser/autofill/form_structure_browsertest.cc

@@ -5,6 +5,7 @@
 #include <algorithm>
 #include <vector>
 
+#include "base/bind.h"
 #include "base/command_line.h"
 #include "base/feature_list.h"
 #include "base/files/file_enumerator.h"
@@ -28,6 +29,10 @@
 #include "components/autofill/core/browser/form_structure.h"
 #include "components/autofill/core/common/autofill_features.h"
 #include "content/public/common/content_switches.h"
+#include "net/http/http_status_code.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
+#include "net/test/embedded_test_server/http_request.h"
+#include "net/test/embedded_test_server/http_response.h"
 #include "url/gurl.h"
 
 #if defined(OS_MACOSX)
@@ -37,21 +42,15 @@
 namespace autofill {
 namespace {
 
+using net::test_server::BasicHttpResponse;
+using net::test_server::HttpRequest;
+using net::test_server::HttpResponse;
+
 const base::FilePath::CharType kTestName[] = FILE_PATH_LITERAL("heuristics");
 
-// Convert the |html| snippet to a data URI.
-GURL HTMLToDataURI(const std::string& html) {
-  // GURL requires data URLs to be UTF-8 and will fail below if it's not.
-  CHECK(base::IsStringUTF8(html)) << "Input file is not UTF-8.";
-
-  // Strip `\n`, `\t`, `\r` from |html| to match old `data:` URL behavior.
-  std::string stripped_html;
-  for (const auto& character : html) {
-    if (character == '\n' || character == '\t' || character == '\r')
-      continue;
-    stripped_html.push_back(character);
-  }
-  return GURL(std::string("data:text/html;charset=utf-8,") + stripped_html);
+const std::set<base::FilePath::StringType>& GetFailingTestNames() {
+  static auto* failing_test_names = new std::set<base::FilePath::StringType>{};
+  return *failing_test_names;
 }
 
 const base::FilePath& GetTestDataDir() {
@@ -65,16 +64,17 @@ const base::FilePath& GetTestDataDir() {
   return dir;
 }
 
-const std::vector<base::FilePath> GetTestFiles() {
-  base::FilePath dir;
-  CHECK(PathService::Get(base::DIR_SOURCE_ROOT, &dir));
-  dir = dir.AppendASCII("components")
-            .AppendASCII("test")
-            .AppendASCII("data")
-            .AppendASCII("autofill")
-            .Append(kTestName)
-            .AppendASCII("input");
-  base::FileEnumerator input_files(dir, false, base::FileEnumerator::FILES);
+const base::FilePath GetInputDir() {
+  static base::FilePath input_dir = GetTestDataDir()
+                                        .AppendASCII("autofill")
+                                        .Append(kTestName)
+                                        .AppendASCII("input");
+  return input_dir;
+}
+
+std::vector<base::FilePath> GetTestFiles() {
+  base::FileEnumerator input_files(GetInputDir(), false,
+                                   base::FileEnumerator::FILES);
   std::vector<base::FilePath> files;
   for (base::FilePath input_file = input_files.Next(); !input_file.empty();
        input_file = input_files.Next()) {
@@ -89,14 +89,20 @@ const std::vector<base::FilePath> GetTestFiles() {
   return files;
 }
 
-const std::set<base::FilePath::StringType>& GetFailingTestNames() {
-  // TODO(crbug.com/789944): Reenable these tests.
-  static std::set<base::FilePath::StringType>* failing_test_names =
-      new std::set<base::FilePath::StringType>{
-          FILE_PATH_LITERAL("067_register_rei.com.html"),
-          FILE_PATH_LITERAL("074_register_threadless.com.html"),
-      };
-  return *failing_test_names;
+std::string FormStructuresToString(
+    const std::vector<std::unique_ptr<FormStructure>>& forms) {
+  std::string forms_string;
+  for (const auto& form : forms) {
+    for (const auto& field : *form) {
+      forms_string += field->Type().ToString();
+      forms_string += " | " + base::UTF16ToUTF8(field->name);
+      forms_string += " | " + base::UTF16ToUTF8(field->label);
+      forms_string += " | " + base::UTF16ToUTF8(field->value);
+      forms_string += " | " + field->section();
+      forms_string += "\n";
+    }
+  }
+  return forms_string;
 }
 
 }  // namespace
@@ -113,21 +119,24 @@ class FormStructureBrowserTest
   ~FormStructureBrowserTest() override;
 
   // InProcessBrowserTest
-  void SetUpCommandLine(base::CommandLine* command_line) override {
-    // Suppress most output logs because we can't really control the output for
-    // arbitrary test sites.
-    command_line->AppendSwitchASCII(switches::kLoggingLevel, "2");
-  }
+  void SetUpCommandLine(base::CommandLine* command_line) override;
+
+  // BrowserTestBase
+  void SetUpOnMainThread() override;
 
   // DataDrivenTest:
   void GenerateResults(const std::string& input, std::string* output) override;
 
-  // Serializes the given |forms| into a string.
-  std::string FormStructuresToString(
-      const std::vector<std::unique_ptr<FormStructure>>& forms);
-
  private:
+  std::unique_ptr<HttpResponse> HandleRequest(const HttpRequest& request);
+
   base::test::ScopedFeatureList feature_list_;
+
+  // The response content to be returned by the embedded test server. Note that
+  // this is populated in the main thread as a part of the setup in the
+  // GenerateResults method but it is consumed later in the IO thread by the
+  // embedded test server to generate the response.
+  std::string html_content_;
   DISALLOW_COPY_AND_ASSIGN(FormStructureBrowserTest);
 };
 
@@ -140,11 +149,42 @@ FormStructureBrowserTest::FormStructureBrowserTest()
 FormStructureBrowserTest::~FormStructureBrowserTest() {
 }
 
+void FormStructureBrowserTest::SetUpCommandLine(
+    base::CommandLine* command_line) {
+  InProcessBrowserTest::SetUpCommandLine(command_line);
+  // Suppress most output logs because we can't really control the output for
+  // arbitrary test sites.
+  command_line->AppendSwitchASCII(switches::kLoggingLevel, "2");
+}
+
+void FormStructureBrowserTest::SetUpOnMainThread() {
+  InProcessBrowserTest::SetUpOnMainThread();
+
+  embedded_test_server()->RegisterRequestHandler(base::BindRepeating(
+      &FormStructureBrowserTest::HandleRequest, base::Unretained(this)));
+  ASSERT_TRUE(embedded_test_server()->Start());
+}
+
 void FormStructureBrowserTest::GenerateResults(const std::string& input,
                                                std::string* output) {
-  ASSERT_NO_FATAL_FAILURE(ui_test_utils::NavigateToURL(browser(),
-                                                       HTMLToDataURI(input)));
+  // Cache the content to be returned by the embedded test server. This data
+  // is readonly after this point.
+  html_content_.clear();
+  html_content_.reserve(input.length());
+  for (const char c : input) {
+    // Strip `\n`, `\t`, `\r` from |html| to match old `data:` URL behavior.
+    // TODO(crbug/239819): the tests expect weird concatenation behavior based
+    //   legacy data URL behavior. Fix this so the the tests better represent
+    //   the parsing being done in the wild.
+    if (c != '\r' && c != '\n' && c != '\t')
+      html_content_.push_back(c);
+  }
 
+  // Navigate to the test html content.
+  ASSERT_NO_FATAL_FAILURE(ui_test_utils::NavigateToURL(
+      browser(), embedded_test_server()->GetURL("/test.html")));
+
+  // Dump the form fields (and their inferred field types).
   content::WebContents* web_contents =
       browser()->tab_strip_model()->GetActiveWebContents();
   ContentAutofillDriver* autofill_driver =
@@ -158,20 +198,13 @@ void FormStructureBrowserTest::GenerateResults(const std::string& input,
   *output = FormStructuresToString(forms);
 }
 
-std::string FormStructureBrowserTest::FormStructuresToString(
-    const std::vector<std::unique_ptr<FormStructure>>& forms) {
-  std::string forms_string;
-  for (const auto& form : forms) {
-    for (const auto& field : *form) {
-      forms_string += field->Type().ToString();
-      forms_string += " | " + base::UTF16ToUTF8(field->name);
-      forms_string += " | " + base::UTF16ToUTF8(field->label);
-      forms_string += " | " + base::UTF16ToUTF8(field->value);
-      forms_string += " | " + field->section();
-      forms_string += "\n";
-    }
-  }
-  return forms_string;
+std::unique_ptr<HttpResponse> FormStructureBrowserTest::HandleRequest(
+    const HttpRequest& request) {
+  auto response = std::make_unique<BasicHttpResponse>();
+  response->set_code(net::HTTP_OK);
+  response->set_content(html_content_);
+  response->set_content_type("text/html; charset=utf-8");
+  return std::move(response);
 }
 
 IN_PROC_BROWSER_TEST_P(FormStructureBrowserTest, DataDrivenHeuristics) {

components/autofill/core/browser/form_structure.cc

@@ -32,6 +32,8 @@
 #include "components/autofill/core/browser/rationalization_util.h"
 #include "components/autofill/core/browser/validation.h"
 #include "components/autofill/core/common/autofill_constants.h"
+#include "components/autofill/core/common/autofill_regex_constants.h"
+#include "components/autofill/core/common/autofill_regexes.h"
 #include "components/autofill/core/common/autofill_util.h"
 #include "components/autofill/core/common/form_data.h"
 #include "components/autofill/core/common/form_data_predictions.h"
@@ -645,10 +647,11 @@ bool FormStructure::ShouldBeParsed() const {
     return false;
   }
 
-  // Rule out http(s)://*/search?...
-  //  e.g. http://www.google.com/search?q=...
-  //       http://search.yahoo.com/search?p=...
-  if (target_url_.path_piece() == "/search") {
+  // Rule out search forms.
+  static const base::string16 kUrlSearchActionPattern =
+      base::UTF8ToUTF16(kUrlSearchActionRe);
+  if (MatchesPattern(base::UTF8ToUTF16(target_url_.path_piece()),
+                     kUrlSearchActionPattern)) {
     return false;
   }
 

components/autofill/core/common/autofill_regex_constants.cc

@@ -297,6 +297,10 @@ const char kPhoneSuffixRe[] = "suffix";
 const char kPhoneExtensionRe[] =
     "\\bext|ext\\b|extension"
     "|ramal";  // pt-BR, pt-PT
+
+/////////////////////////////////////////////////////////////////////////////
+// validation.cc
+/////////////////////////////////////////////////////////////////////////////
 const char kUPIVirtualPaymentAddressRe[] =
     "^\\w+@("
     "allbank|"      // Allahabad Bank UPI
@@ -348,4 +352,9 @@ const char kUPIVirtualPaymentAddressRe[] =
     "yesbank"       // NuPay
     ")$";
 
+/////////////////////////////////////////////////////////////////////////////
+// form_structure.cc
+/////////////////////////////////////////////////////////////////////////////
+const char kUrlSearchActionRe[] = "/search(/|((\\w*\\.\\w+)?$))";
+
 }  // namespace autofill

components/autofill/core/common/autofill_regex_constants.h

@@ -62,6 +62,14 @@ extern const char kPhoneExtensionRe[];
 //   - https://upipayments.co.in/virtual-payment-address-vpa/
 extern const char kUPIVirtualPaymentAddressRe[];
 
+// Match the path values for form actions that look like generic search:
+//  e.g. /search
+//       /search/
+//       /search/products...
+//       /products/search/
+//       /blah/search_all.jsp
+extern const char kUrlSearchActionRe[];
+
 }  // namespace autofill
 
 #endif  // COMPONENTS_AUTOFILL_CORE_BROWSER_AUTOFILL_REGEX_CONSTANTS_H_

components/test/data/autofill/heuristics/output/018_checkout_ae.com.out

@@ -3,10 +3,6 @@ UNKNOWN_TYPE | logonPassword | password | password | logonUsername_1-default
 UNKNOWN_TYPE | zip | Find a store | Enter ZIP/Postal | zip_1-default
 UNKNOWN_TYPE | username | Access your wish list | Email | username_1-default
 UNKNOWN_TYPE | password | Password | Password | username_1-default
-NAME_FIRST | firstName | Search wish lists by name | First Name | firstName_1-default
-NAME_LAST | lastName | Last Name | Last Name | firstName_1-default
-EMAIL_ADDRESS | email | Or search wish lists by email | Email | firstName_1-default
-UNKNOWN_TYPE | /aeo/commerce/search/formhandlers/AEOQueryFormHandler.searchRequest.question | Keyword or Style # | Keyword or Style # | /aeo/commerce/search/formhandlers/AEOQueryFormHandler.searchRequest.question_1-default
 ADDRESS_HOME_COUNTRY | countryType | Country Type | usa | countryType_1-default
 ADDRESS_HOME_COUNTRY | country | Country, APO/FPO | US | countryType_1-default
 NAME_FIRST | firstName | First Name |  | countryType_1-default

components/test/data/autofill/heuristics/output/035_checkout_petco.com.out

-UNKNOWN_TYPE | zip | Your ZIP Code | Your ZIP Code | zip_1-default
-UNKNOWN_TYPE | Ntt | Search PETCO.com | Search PETCO.com | Ntt_1-default
 UNKNOWN_TYPE | txtEmail | Your Email Address | Your Email Address | txtEmail_1-default
 NAME_FIRST | ctl00$ctl00$cphBody$cphBody$txtSA_FirstName | First Name * |  | ctl00$ctl00$cphBody$cphBody$txtSA_FirstName_1-default
 NAME_LAST | ctl00$ctl00$cphBody$cphBody$txtSA_LastName | Last Name * |  | ctl00$ctl00$cphBody$cphBody$txtSA_FirstName_1-default

components/test/data/autofill/heuristics/output/036_checkout_petsmart.com.out

-UNKNOWN_TYPE | kw | search petsmart.com | search petsmart.com | kw_1-default
 ADDRESS_HOME_COUNTRY | billCountry | • Country: | US | billCountry_1-default
 NAME_FIRST | billFname | • First Name: |  | billCountry_1-default
 NAME_LAST | billLname | • Last Name: |  | billCountry_1-default

components/test/data/autofill/heuristics/output/040_checkout_urbanoutfitters.com.out

-UNKNOWN_TYPE | searchPhrase |  |  | searchPhrase_1-default
 UNKNOWN_TYPE | shipto | Select a Saved Shipping Address |  | shipto_1-default
 NAME_FIRST | shippingFirstName | First Name | First Name | shipto_1-default
 NAME_LAST | shippingLastName | Last Name | Last Name | shipto_1-default

components/test/data/autofill/heuristics/output/042_checkout_williams-sonoma.com.out

-UNKNOWN_TYPE | words | Search by keyword, recipe or item # | Enter keyword, recipe or item # | words_1-default
 NAME_FULL | shipTos[0].address.fullName | Full Name* |  | shipTos[0].address.fullName_1-default
 ADDRESS_HOME_LINE1 | shipTos[0].address.addrLine1 | Address* |  | shipTos[0].address.fullName_1-default
 ADDRESS_HOME_LINE2 | shipTos[0].address.addrLine2 | Address Line 2 |  | shipTos[0].address.fullName_1-default

components/test/data/autofill/heuristics/output/059_register_macys.com.out

-UNKNOWN_TYPE | Keyword |  |  | Keyword_1-default
 NAME_FIRST | FirstName | First Name |  | FirstName_1-default
 NAME_LAST | LastName | Last Name |  | FirstName_1-default
 ADDRESS_HOME_LINE1 | Address1 | Address |  | FirstName_1-default

components/test/data/autofill/heuristics/output/060_register_mcphee.com.out

-UNKNOWN_TYPE | search_query | Search |  | search_query_1-default
 EMAIL_ADDRESS | FormField[1][1] | * Email Address: |  | FormField[1][1]_1-default
 UNKNOWN_TYPE | FormField[1][2] | * Password: |  | FormField[1][1]_1-default
 UNKNOWN_TYPE | FormField[1][3] | * Confirm Password: |  | FormField[1][1]_1-default

components/test/data/autofill/heuristics/output/061_register_myspace.com.out

-UNKNOWN_TYPE | q | Search People |  | q_1-default
 UNKNOWN_TYPE | accountType | Personal | 2 | accountType_1-default
 UNKNOWN_TYPE | accountType | Musician | 7 | accountType_1-default
 UNKNOWN_TYPE | accountType | Comedian | 15 | accountType_1-default

components/test/data/autofill/heuristics/output/063_register_officedepot.com.out

 UNKNOWN_TYPE | zip |  |  | zip_1-default
-UNKNOWN_TYPE | Ntt | Search |  | Ntt_1-default
 NAME_FIRST | addrsForm[0].firstName | *First Name: |  | addrsForm[0].firstName_1-default
 NAME_MIDDLE_INITIAL | addrsForm[0].middleInitial | Middle Initial: |  | addrsForm[0].firstName_1-default
 NAME_LAST | addrsForm[0].lastName | *Last Name: |  | addrsForm[0].firstName_1-default

components/test/data/autofill/heuristics/output/064_register_officemax.com.out

@@ -6,4 +6,3 @@ EMAIL_ADDRESS | /atg/userprofiling/ProfileFormHandler.confirmEmailAddress | * Co
 ADDRESS_HOME_ZIP | /atg/userprofiling/ProfileFormHandler.value.zip | * Zip: |  | /atg/userprofiling/ProfileFormHandler.value.maxPerks_1-default
 UNKNOWN_TYPE | /atg/userprofiling/ProfileFormHandler.value.password | * Password: |  | /atg/userprofiling/ProfileFormHandler.value.maxPerks_1-default
 UNKNOWN_TYPE | /atg/userprofiling/ProfileFormHandler.value.confirmPassword | * Confirm Password: |  | /atg/userprofiling/ProfileFormHandler.value.maxPerks_1-default
-UNKNOWN_TYPE | freeText | Search by Keyword or Item # | Search by Keyword or Item # | freeText_1-default

components/test/data/autofill/heuristics/output/065_register_pyramidcollection.com.out

-UNKNOWN_TYPE | q | search | enter keyword(s) or item number | q_1-default
 EMAIL_ADDRESS | E1 | Email Address * |  | E1_1-default
 NAME_FIRST | F1 | First Name * |  | E1_1-default
 NAME_LAST | L1 | Last Name * |  | E1_1-default

components/test/data/autofill/heuristics/output/071_register_sourceforge.net.out

-UNKNOWN_TYPE | q | Find Open Source Software |  | q_1-default
 NAME_FULL | X1mRVeMqejLnZpd1etxNGHllat2M | Name: |  | X1mRVeMqejLnZpd1etxNGHllat2M_1-default
 EMAIL_ADDRESS | X129ZdMbfixhIflk8_zHDFWB72qk | Email: |  | X1mRVeMqejLnZpd1etxNGHllat2M_1-default
 UNKNOWN_TYPE | X2n9HcN3dx1-mSbLywp_L-szMydw | Username: |  | X1mRVeMqejLnZpd1etxNGHllat2M_1-default