[iOS] Extend CWTChromeDriver to support fuzzing

CWTChromeDriver contains a partial implementation of the WebDriver
protocol for browser automation, used for Web Platform Tests.

This CL adds support for a new "chrome_crashtest" command, that can
be used for fuzzing. This new command is similar to the existing
"url" command, that loads a URL and waits for it to finish loading.
However, the new command waits for an additional amount of time
(specified as an argument) after page load to catch cases where a
test page crashes just after page load is complete rather than
during load.

This new command also extracts the stderr output of the app process
and returns it to the caller. This is needed for fuzzing because
ASan crash stacks will be sent to the app process' stderr before
the app process is terminated due to a bug found by ASan.

Finally, this new command supports "file:///" URLs by serving the
given file using an EmbeddedTestServer, since Chrome on iOS does
not support directly loading such URLs. This is useful for fuzzing
since fuzzing test cases are typically provided as a local HTML
file.

CWTChromeDriver is not shipped to end users (unlike ChromeDriver on
non-iOS platforms), so supporting a non-standard command for test
purposes does not create any compatibility issues.

Change-Id: I84ad0100e40c12065a18325501d997a5b34220dd
Bug: 1158540
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2600023Reviewed-by: Olivier Robin <olivierrobin@chromium.org>
Commit-Queue: Ali Juma <ajuma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#846117}

ios/chrome/test/wpt/BUILD.gn

@@ -80,6 +80,8 @@ source_set("app_support") {
   configs += [ "//build/config/compiler:enable_arc" ]
 
   sources = [
+    "cwt_stderr_logger.h",
+    "cwt_stderr_logger.mm",
     "cwt_tests_hook.mm",
     "cwt_webdriver_app_interface.mm",
   ]

ios/chrome/test/wpt/cwt_request_handler.h

@@ -8,10 +8,12 @@
 #import <Foundation/Foundation.h>
 #include <string>
 
+#include "base/files/file_path.h"
 #include "base/ios/block_types.h"
 #include "base/macros.h"
 #include "base/optional.h"
 #include "base/values.h"
+#include "net/test/embedded_test_server/embedded_test_server.h"
 #include "net/test/embedded_test_server/http_request.h"
 #include "net/test/embedded_test_server/http_response.h"
 
@@ -40,6 +42,8 @@ class CWTRequestHandler {
   // closed.
   CWTRequestHandler(ProceduralBlock sesssion_completion_handler);
 
+  ~CWTRequestHandler();
+
   // Creates responses for HTTP requests according to the WebDriver protocol.
   std::unique_ptr<net::test_server::HttpResponse> HandleRequest(
       const net::test_server::HttpRequest& request);
@@ -56,6 +60,11 @@ class CWTRequestHandler {
   // complete.
   base::Value NavigateToUrl(const base::Value* url);
 
+  // Navigates the target tab to the URL given in |input|, waits for the page
+  // load to complete, and then waits for the additional time specified in
+  // |input|. Returns the stderr output produced by the app during page load.
+  base::Value NavigateToUrlForCrashTest(const base::Value& input);
+
   // Sets timeouts used when performing browser operations.
   base::Value SetTimeouts(const base::Value& timeouts);
 
@@ -124,6 +133,12 @@ class CWTRequestHandler {
   NSTimeInterval script_timeout_;
   NSTimeInterval page_load_timeout_;
 
+  // A server for test files used in crash tests.
+  net::EmbeddedTestServer test_case_server_;
+
+  // The directory used for test files for crash tests.
+  base::FilePath test_case_directory_;
+
   DISALLOW_COPY_AND_ASSIGN(CWTRequestHandler);
 };
 

ios/chrome/test/wpt/cwt_request_handler.mm

@@ -4,11 +4,16 @@
 
 #import "ios/chrome/test/wpt/cwt_request_handler.h"
 
+#import <XCTest/XCTest.h>
+
 #include "base/debug/stack_trace.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
 #include "base/guid.h"
 #include "base/json/json_reader.h"
 #include "base/json/json_writer.h"
 #include "base/strings/sys_string_conversions.h"
+#include "base/test/ios/wait_util.h"
 #include "components/version_info/version_info.h"
 #import "ios/chrome/test/wpt/cwt_constants.h"
 #import "ios/chrome/test/wpt/cwt_webdriver_app_interface.h"
@@ -40,6 +45,17 @@ const char kWebDriverAsyncScriptCommand[] = "async";
 const char kWebDriverScreenshotCommand[] = "screenshot";
 const char kWebDriverWindowRectCommand[] = "rect";
 
+// Non-standard commands used only for testing Chrome.
+// This command is similar to the standard "url" command. It loads the URL
+// specified by the kWebDriverURLRequestField argument, waits up till the
+// currently-set page load time (see the standard "timeouts" command) for the
+// page to finish loading, but then waits an additional amount of time
+// (specified in seconds by the kChromeCrashWaitTime argument) for the page to
+// crash. It then returns the stderr produced by the app during this time, in
+// the kChromeStderrValueField response field. If the given URL is a file URL,
+// the given file is copied and served from a local EmbeddedTestServer.
+const char kChromeCrashTestCommand[] = "chrome_crashtest";
+
 // WebDriver error codes.
 const char kWebDriverInvalidArgumentError[] = "invalid argument";
 const char kWebDriverInvalidSessionError[] = "invalid session id";
@@ -66,6 +82,13 @@ const char kWebDriverNoMatchingWindowMessage[] =
 const char kWebDriverMissingScriptMessage[] = "No script argument";
 const char kWebDriverScriptTimeoutMessage[] = "Script execution timed out";
 
+// Non-standard error messages, used only for testing Chrome.
+const char kChromeInvalidExtraWaitMessage[] =
+    "Extra wait must be a non-negative integer";
+const char kChromeInvalidUrlMessage[] = "The provided URL is not valid";
+const char kChromeFileCannotBeCopiedMessage[] =
+    "The provided input file cannot be copied";
+
 // WebDriver request field names. These are fields that are contained within
 // the body of a POST request.
 const char kWebDriverURLRequestField[] = "url";
@@ -74,6 +97,11 @@ const char kWebDriverPageLoadTimeoutRequestField[] = "pageLoad";
 const char kWebDriverWindowHandleRequestField[] = "handle";
 const char kWebDriverScriptRequestField[] = "script";
 
+// Non-standard request field names, used only for testing Chrome.
+// The additional time (in seconds) to wait for a crash after a successful page
+// load.
+const char kChromeCrashWaitTime[] = "chrome_crashWaitTime";
+
 // WebDriver response field name. This is the top-level field in the JSON object
 // contained in a response.
 const char kWebDriverValueResponseField[] = "value";
@@ -87,6 +115,10 @@ const char kWebDriverErrorMessageValueField[] = "message";
 const char kWebDriverSessionIdValueField[] = "sessionId";
 const char kWebDriverStackTraceValueField[] = "stacktrace";
 
+// Non-standard value field names, used only when testing Chrome.
+// Stderr output from the app.
+const char kChromeStderrValueField[] = "chrome_stderr";
+
 // Field names for the "capabilities" struct that's included in the response
 // when creating a session.
 const char kCapabilitiesBrowserNameField[] = "browserName";
@@ -124,7 +156,16 @@ bool IsErrorValue(const base::Value& value) {
 CWTRequestHandler::CWTRequestHandler(ProceduralBlock session_completion_handler)
     : session_completion_handler_(session_completion_handler),
       script_timeout_(kDefaultScriptTimeout),
-      page_load_timeout_(kDefaultPageLoadTimeout) {}
+      page_load_timeout_(kDefaultPageLoadTimeout) {
+  base::CreateNewTempDirectory(base::FilePath::StringType(),
+                               &test_case_directory_);
+  test_case_server_.ServeFilesFromDirectory(test_case_directory_);
+  if (!test_case_server_.Start()) {
+    XCTFail("Unable to start test case server.");
+  }
+}
+
+CWTRequestHandler::~CWTRequestHandler() = default;
 
 base::Optional<base::Value> CWTRequestHandler::ProcessCommand(
     const std::string& command,
@@ -164,6 +205,9 @@ base::Optional<base::Value> CWTRequestHandler::ProcessCommand(
                               kWebDriverNoActiveSessionMessage);
     }
 
+    if (command == kChromeCrashTestCommand)
+      return NavigateToUrlForCrashTest(*content);
+
     if (command == kWebDriverNavigationCommand)
       return NavigateToUrl(content->FindKey(kWebDriverURLRequestField));
 
@@ -296,6 +340,81 @@ base::Value CWTRequestHandler::NavigateToUrl(const base::Value* url) {
                           kWebDriverPageLoadTimeoutMessage);
 }
 
+base::Value CWTRequestHandler::NavigateToUrlForCrashTest(
+    const base::Value& input) {
+  const base::Value* url_str = input.FindKey(kWebDriverURLRequestField);
+  if (!url_str || !url_str->is_string()) {
+    return CreateErrorValue(kWebDriverInvalidArgumentError,
+                            kWebDriverMissingURLMessage);
+  }
+
+  GURL url(url_str->GetString());
+  if (!url.is_valid()) {
+    return CreateErrorValue(kWebDriverInvalidArgumentError,
+                            kChromeInvalidUrlMessage);
+  }
+
+  if (url.SchemeIsFile()) {
+    // Copy the file to the directory being served by the test server.
+    base::FilePath test_input_file(url.GetContent());
+    base::FilePath test_destination_file =
+        test_case_directory_.Append(test_input_file.BaseName());
+    bool copied_file = base::CopyFile(test_input_file, test_destination_file);
+    if (!copied_file) {
+      return CreateErrorValue(kWebDriverInvalidArgumentError,
+                              kChromeFileCannotBeCopiedMessage);
+    }
+    url = test_case_server_.GetURL("/" + test_input_file.BaseName().value());
+  }
+
+  base::FilePath log_file;
+  base::CreateTemporaryFile(&log_file);
+  [CWTWebDriverAppInterface
+      logStderrToFilePath:base::SysUTF8ToNSString(log_file.value())];
+
+  // Once the test page is loaded, the app might crash at any time until the
+  // tab is closed. Re-launch the app if it crashes.
+  @try {
+    NSError* error = [CWTWebDriverAppInterface
+                 loadURL:base::SysUTF8ToNSString(url.spec())
+                   inTab:base::SysUTF8ToNSString(target_tab_id_)
+        timeoutInSeconds:page_load_timeout_];
+
+    if (!error) {
+      const base::Value* extra_wait = input.FindKey(kChromeCrashWaitTime);
+      if (extra_wait) {
+        if (!extra_wait->is_int() || extra_wait->GetInt() < 0) {
+          return CreateErrorValue(kWebDriverInvalidArgumentError,
+                                  kChromeInvalidExtraWaitMessage);
+        }
+        base::test::ios::SpinRunLoopWithMinDelay(
+            base::TimeDelta::FromSeconds(extra_wait->GetInt()));
+      }
+    }
+
+    [CWTWebDriverAppInterface openNewTab];
+    [CWTWebDriverAppInterface
+        closeTabWithID:base::SysUTF8ToNSString(target_tab_id_)];
+    target_tab_id_ =
+        base::SysNSStringToUTF8([CWTWebDriverAppInterface currentTabID]);
+
+    [CWTWebDriverAppInterface stopLoggingStderr];
+  } @catch (NSException* exception) {
+    dispatch_sync(dispatch_get_main_queue(), ^{
+      [[[XCUIApplication alloc] init] launch];
+    });
+    target_tab_id_ =
+        base::SysNSStringToUTF8([CWTWebDriverAppInterface currentTabID]);
+  }
+
+  std::string stderr_contents;
+  base::ReadFileToString(log_file, &stderr_contents);
+
+  base::Value result(base::Value::Type::DICTIONARY);
+  result.SetStringKey(kChromeStderrValueField, stderr_contents);
+  return result;
+}
+
 base::Value CWTRequestHandler::SetTimeouts(const base::Value& timeouts) {
   for (const auto& timeout : timeouts.DictItems()) {
     if (!timeout.second.is_int() || timeout.second.GetInt() < 0) {

ios/chrome/test/wpt/cwt_stderr_logger.h

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IOS_CHROME_TEST_WPT_CWT_STDERR_LOGGER_H_
+#define IOS_CHROME_TEST_WPT_CWT_STDERR_LOGGER_H_
+
+#include "base/memory/singleton.h"
+
+namespace base {
+class FilePath;
+}
+
+// This class manages redirecting stderr output to a file. Redirection can be
+// started and stopped multiple times, but there must be a call to
+// StopRedirectingToFile() between any two calls to StartRedirectingToFile().
+class CWTStderrLogger {
+ public:
+  // Returns the singleton instance of this class.
+  static CWTStderrLogger* GetInstance();
+
+  CWTStderrLogger(const CWTStderrLogger&) = delete;
+  CWTStderrLogger& operator=(const CWTStderrLogger&) = delete;
+
+  // Starts redirecting stderr output to the file with the given path. This
+  // file must already exist. Any existing content will not be overwritten;
+  // instead, new content is appended to the file..
+  void StartRedirectingToFile(const base::FilePath& file_path);
+
+  // Stops redirecting stderr output to a file.
+  void StopRedirectingToFile();
+
+ private:
+  friend struct base::DefaultSingletonTraits<CWTStderrLogger>;
+
+  CWTStderrLogger() = default;
+
+  // When redirection is active, this saves a copy of the old stderr file
+  // descriptor, which is restored as the destination of stderr after
+  // redirection is stopped.
+  int saved_stderr_file_descriptor_ = -1;
+
+  // While redirection is active, this stores the file descriptor for the
+  // destination file.
+  int redirection_destination_file_descriptor_ = -1;
+};
+
+#endif  // IOS_CHROME_TEST_WPT_CWT_STDERR_LOGGER_H_

ios/chrome/test/wpt/cwt_stderr_logger.mm

+// Copyright 2021 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "ios/chrome/test/wpt/cwt_stderr_logger.h"
+
+#include "base/files/file.h"
+#include "base/files/file_path.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+CWTStderrLogger* CWTStderrLogger::GetInstance() {
+  return base::Singleton<CWTStderrLogger>::get();
+}
+
+void CWTStderrLogger::StartRedirectingToFile(const base::FilePath& file_path) {
+  base::File destination_file(file_path,
+                              base::File::FLAG_OPEN | base::File::FLAG_APPEND);
+  DCHECK_EQ(saved_stderr_file_descriptor_, -1);
+  saved_stderr_file_descriptor_ = dup(STDERR_FILENO);
+  redirection_destination_file_descriptor_ =
+      destination_file.TakePlatformFile();
+  dup2(redirection_destination_file_descriptor_, STDERR_FILENO);
+}
+
+void CWTStderrLogger::StopRedirectingToFile() {
+  DCHECK_NE(saved_stderr_file_descriptor_, -1);
+  close(redirection_destination_file_descriptor_);
+  redirection_destination_file_descriptor_ = -1;
+
+  // Reset stderr to its previous destination, before redirection began.
+  dup2(saved_stderr_file_descriptor_, STDERR_FILENO);
+  close(saved_stderr_file_descriptor_);
+  saved_stderr_file_descriptor_ = -1;
+}

ios/chrome/test/wpt/cwt_webdriver_app_interface.h

@@ -32,6 +32,9 @@
 // tab.
 + (NSError*)closeTabWithID:(NSString*)ID;
 
+// Opens a new tab, makes this tab the current tab, and return its id.
++ (NSString*)openNewTab;
+
 // Makes the tab identified by |ID| the current tab. Returns an error if there
 // is no such tab.
 + (NSError*)switchToTabWithID:(NSString*)ID;
@@ -52,6 +55,12 @@
 // encoded image. If no such tab exists, returns nil.
 + (NSString*)takeSnapshotOfTabWithID:(NSString*)ID;
 
+// Starts redirecting stderr output to the file with the given path.
++ (void)logStderrToFilePath:(NSString*)filePath;
+
+// Stops redirecting stderr output to a file.
++ (void)stopLoggingStderr;
+
 @end
 
 #endif  // IOS_CHROME_TEST_WPT_CWT_WEBDRIVER_APP_INTERFACE_H_

ios/chrome/test/wpt/cwt_webdriver_app_interface.mm

@@ -4,6 +4,8 @@
 
 #import "ios/chrome/test/wpt/cwt_webdriver_app_interface.h"
 
+#include "base/files/file.h"
+#include "base/files/file_path.h"
 #include "base/json/json_writer.h"
 #include "base/strings/stringprintf.h"
 #include "base/strings/sys_string_conversions.h"
@@ -17,6 +19,7 @@
 #import "ios/chrome/test/app/chrome_test_util.h"
 #import "ios/chrome/test/app/settings_test_util.h"
 #import "ios/chrome/test/app/tab_test_util.h"
+#import "ios/chrome/test/wpt/cwt_stderr_logger.h"
 #import "ios/testing/nserror_util.h"
 #import "ios/web/public/test/navigation_test_util.h"
 #import "ios/web/public/ui/crw_web_view_proxy.h"
@@ -157,6 +160,16 @@ void DispatchSyncOnMainThread(void (^block)(void)) {
   return error;
 }
 
++ (NSString*)openNewTab {
+  __block NSString* tabID = nil;
+  DispatchSyncOnMainThread(^{
+    chrome_test_util::OpenNewTab();
+    tabID = GetIdForWebState(chrome_test_util::GetCurrentWebState());
+  });
+
+  return tabID;
+}
+
 + (NSError*)switchToTabWithID:(NSString*)ID {
   __block NSError* error = nil;
   DispatchSyncOnMainThread(^{
@@ -290,4 +303,13 @@ void DispatchSyncOnMainThread(void (^block)(void)) {
   return [snapshotAsPNG base64EncodedStringWithOptions:0];
 }
 
++ (void)logStderrToFilePath:(NSString*)filePath {
+  base::FilePath stderrPath(base::SysNSStringToUTF8(filePath));
+  CWTStderrLogger::GetInstance()->StartRedirectingToFile(stderrPath);
+}
+
++ (void)stopLoggingStderr {
+  CWTStderrLogger::GetInstance()->StopRedirectingToFile();
+}
+
 @end