[ios] Adds code to verify that tests are running with custom webkit.

If --run-with-custom-webkit is passed on the commandline, WebTestSuite
(gtest) and ChromeTestCase (egtest) will verify that the custom-built
WebKit frameworks were loaded, as opposed to the system-provided
frameworks.  This will provide a loud warning when the frameworks cannot
be loaded, since otherwise tests would pass silently using the system
frameworks, which is not what we want.

BUG=934252

Change-Id: I684e9ced303d36d1019714e3c807862eb2803f5a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1850011
Commit-Queue: Rohit Rao <rohitrao@chromium.org>
Reviewed-by: Justin Cohen <justincohen@chromium.org>
Cr-Commit-Position: refs/heads/master@{#704286}

ios/chrome/test/earl_grey/BUILD.gn

@@ -266,6 +266,7 @@ source_set("test_support") {
     "//ios/chrome/browser/ui/toolbar/public",
     "//ios/chrome/browser/ui/util",
     "//ios/chrome/test/app:test_support",
+    "//ios/testing:verify_custom_webkit",
     "//ios/testing/earl_grey:earl_grey_support",
     "//ios/third_party/material_components_ios",
     "//ios/third_party/webkit",
@@ -374,6 +375,7 @@ source_set("eg_app_support+eg2") {
     "//ios/chrome/browser/ui/util",
     "//ios/chrome/test/app:test_support",
     "//ios/testing:nserror_support",
+    "//ios/testing:verify_custom_webkit",
     "//ios/testing/earl_grey:eg_app_support+eg2",
     "//ios/third_party/earl_grey2:app_framework+link",
     "//ios/third_party/gtx:gtx+link",

ios/chrome/test/earl_grey/chrome_earl_grey.h

@@ -384,6 +384,11 @@ id ExecuteJavaScript(NSString* javascript, NSError* __autoreleasing* out_error);
 // Returns YES if CreditCardScanner feature is enabled.
 - (BOOL)isCreditCardScannerEnabled WARN_UNUSED_RESULT;
 
+// Returns YES if custom WebKit frameworks were properly loaded, rather than
+// system frameworks. Always returns YES if the app was not requested to run
+// with custom WebKit frameworks.
+- (BOOL)isCustomWebKitLoadedIfRequested WARN_UNUSED_RESULT;
+
 #pragma mark - Popup Blocking
 
 // Gets the current value of the popup content setting preference for the

ios/chrome/test/earl_grey/chrome_earl_grey.mm

@@ -647,6 +647,10 @@ GREY_STUB_CLASS_IN_APP_MAIN_QUEUE(ChromeEarlGreyAppInterface)
   return [ChromeEarlGreyAppInterface isCreditCardScannerEnabled];
 }
 
+- (BOOL)isCustomWebKitLoadedIfRequested {
+  return [ChromeEarlGreyAppInterface isCustomWebKitLoadedIfRequested];
+}
+
 #pragma mark - ScopedBlockPopupsPref
 
 - (ContentSetting)popupPrefValue {

ios/chrome/test/earl_grey/chrome_earl_grey_app_interface.h

@@ -313,6 +313,11 @@
 // Returns YES if CreditCardScanner feature is enabled.
 + (BOOL)isCreditCardScannerEnabled WARN_UNUSED_RESULT;
 
+// Returns YES if custom WebKit frameworks were properly loaded, rather than
+// system frameworks. Always returns YES if the app was not requested to run
+// with custom WebKit frameworks.
++ (BOOL)isCustomWebKitLoadedIfRequested WARN_UNUSED_RESULT;
+
 #pragma mark - Popup Blocking
 
 // Gets the current value of the popup content setting preference for the

ios/chrome/test/earl_grey/chrome_earl_grey_app_interface.mm

@@ -25,6 +25,7 @@
 #import "ios/chrome/test/app/tab_test_util.h"
 #import "ios/chrome/test/earl_grey/accessibility_util.h"
 #import "ios/testing/nserror_util.h"
+#include "ios/testing/verify_custom_webkit.h"
 #import "ios/web/common/features.h"
 #import "ios/web/public/deprecated/crw_js_injection_receiver.h"
 #import "ios/web/public/navigation/navigation_manager.h"
@@ -537,6 +538,10 @@ using chrome_test_util::BrowserCommandDispatcherForMainBVC;
   return base::FeatureList::IsEnabled(kCreditCardScanner);
 }
 
++ (BOOL)isCustomWebKitLoadedIfRequested {
+  return IsCustomWebKitLoadedIfRequested();
+}
+
 #pragma mark - ScopedBlockPopupsPref
 
 + (ContentSetting)popupPrefValue {

ios/chrome/test/earl_grey/chrome_test_case.mm

@@ -408,6 +408,8 @@ GREY_STUB_CLASS_IN_APP_MAIN_QUEUE(ChromeTestCaseAppInterface)
 // Dismisses and revert browser settings to default.
 // It also starts the HTTP server and enables mock authentication.
 + (void)setUpHelper {
+  XCTAssertTrue([ChromeEarlGrey isCustomWebKitLoadedIfRequested]);
+
   [CoverageUtils configureCoverageReportPath];
 
   [[self class] startHTTPServer];

ios/testing/BUILD.gn

@@ -12,6 +12,18 @@ group("all_tests") {
   ]
 }
 
+source_set("verify_custom_webkit") {
+  configs += [ "//build/config/compiler:enable_arc" ]
+  testonly = true
+  sources = [
+    "verify_custom_webkit.h",
+    "verify_custom_webkit.mm",
+  ]
+  deps = [
+    "//base",
+  ]
+}
+
 source_set("nserror_support") {
   configs += [ "//build/config/compiler:enable_arc" ]
   testonly = true

ios/testing/verify_custom_webkit.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef IOS_TESTING_VERIFY_CUSTOM_WEBKIT_H_
+#define IOS_TESTING_VERIFY_CUSTOM_WEBKIT_H_
+
+// Returns whether custom WebKit frameworks were loaded if
+// --run-with-custom-webkit was passed on the commandline.  Otherwise, always
+// returns true.
+bool IsCustomWebKitLoadedIfRequested();
+
+#endif  // IOS_TESTING_VERIFY_CUSTOM_WEBKIT_H_

ios/testing/verify_custom_webkit.mm

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#import "ios/testing/verify_custom_webkit.h"
+
+#import <Foundation/Foundation.h>
+#include <mach-o/dyld.h>
+
+#include "base/command_line.h"
+
+#if !defined(__has_feature) || !__has_feature(objc_arc)
+#error "This file requires ARC support."
+#endif
+
+namespace {
+
+// The switch used when running with custom WebKit frameworks.
+const char kRunWithCustomWebKit[] = "run-with-custom-webkit";
+
+}
+
+bool IsCustomWebKitLoadedIfRequested() {
+  if (!base::CommandLine::ForCurrentProcess()->HasSwitch(
+          kRunWithCustomWebKit)) {
+    return true;
+  }
+
+  bool foundIncorrectLoadLocation = false;
+  NSArray<NSString*>* frameworks = @[
+    @"JavaScriptCore.framework/JavaScriptCore",
+    @"WebCore.framework/WebCore",
+    @"WebKit.framework/WebKit",
+    @"WebKitLegacy.framework/WebKitLegacy",
+  ];
+
+  uint32_t numImages = _dyld_image_count();
+  for (uint32_t i = 0; i < numImages; i++) {
+    NSString* imagePath =
+        [NSString stringWithUTF8String:_dyld_get_image_name(i)];
+    for (NSString* framework in frameworks) {
+      if ([imagePath containsString:framework]) {
+        // Custom frameworks are bundled inside a "WebKitFrameworks"
+        // subdirectory, so look for that string to be present in |full_path|.
+        if (![imagePath containsString:@"WebKitFrameworks"]) {
+          // This framework was loaded from an unexpected location.
+          NSLog(@"Unexpectedly loaded %@ from %@ ", framework, imagePath);
+          foundIncorrectLoadLocation = true;
+        }
+      }
+    }
+  }
+
+  // Note that checks are not performed on frameworks that were never loaded at
+  // all.  This function checks that *if* they were loaded, they were loaded
+  // from the correct location.
+  return !foundIncorrectLoadLocation;
+}

ios/web/test/BUILD.gn

@@ -58,6 +58,7 @@ source_set("test_support") {
     ":test_constants",
     "//base",
     "//base/test:test_support",
+    "//ios/testing:verify_custom_webkit",
     "//ios/web",
     "//ios/web/common:web_view_creation_util",
     "//ios/web/navigation:core",

ios/web/test/web_test_suite.mm

@@ -4,8 +4,10 @@
 
 #include "ios/web/public/test/web_test_suite.h"
 
+#include "base/logging.h"
 #include "base/memory/ptr_util.h"
 #include "base/path_service.h"
+#include "ios/testing/verify_custom_webkit.h"
 #include "ios/web/public/navigation/url_schemes.h"
 #import "ios/web/public/test/fakes/test_web_client.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ -19,9 +21,15 @@ namespace web {
 
 WebTestSuite::WebTestSuite(int argc, char** argv)
     : base::TestSuite(argc, argv),
-      web_client_(base::WrapUnique(new TestWebClient)) {}
+      web_client_(base::WrapUnique(new TestWebClient)) {
+  CHECK(IsCustomWebKitLoadedIfRequested());
+}
 
-WebTestSuite::~WebTestSuite() {}
+WebTestSuite::~WebTestSuite() {
+  // Verify again at the end of the test run, in case some frameworks were not
+  // yet loaded when the constructor ran.
+  CHECK(IsCustomWebKitLoadedIfRequested());
+}
 
 void WebTestSuite::Initialize() {
   base::TestSuite::Initialize();