Move data in //chrome/test/data/safe_browsing/dmg/ to CIPD.

Generating the test data at build time is causing issues on certain
bots. The files are too large to commit to source control, so store them
in the CIPD package chromium/chrome/test_data/safe_browsing/dmg.

Bug: 817663
Change-Id: Iebf45db581e2bfdefab0eda32722f81cf036d900
Reviewed-on: https://chromium-review.googlesource.com/1175041Reviewed-by: Jialiu Lin <jialiul@chromium.org>
Reviewed-by: Ken Rockot <rockot@chromium.org>
Reviewed-by: Vadim Shtayura <vadimsh@chromium.org>
Commit-Queue: Robert Sesek <rsesek@chromium.org>
Cr-Commit-Position: refs/heads/master@{#584137}

DEPS

@@ -253,6 +253,17 @@ deps = {
   'src/chrome/test/data/perf/frame_rate/content':
     Var('chromium_git') + '/chromium/frame_rate/content.git' + '@' + 'c10272c88463efeef6bb19c9ec07c42bc8fe22b9',
 
+  'src/chrome/test/data/safe_browsing/dmg': {
+    'packages': [
+      {
+        'package': 'chromium/chrome/test/data/safe_browsing/dmg',
+        'version': 'version:20180816.2',
+      },
+    ],
+    'condition': 'checkout_mac',
+    'dep_type': 'cipd',
+  },
+
   'src/chrome/test/data/xr/webvr_info':
     Var('chromium_git') + '/external/github.com/toji/webvr.info.git' + '@' + 'c58ae99b9ff9e2aa4c524633519570bf33536248',
 

chrome/browser/safe_browsing/download_protection/disk_image_type_sniffer_mac_unittest.cc

@@ -36,9 +36,10 @@ class DiskImageTypeSnifferMacTest
  protected:
   base::FilePath GetFilePath(const char* file_name) {
     base::FilePath test_data;
-    EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &test_data));
-    return test_data.AppendASCII("chrome")
-        .AppendASCII("safe_browsing_dmg")
+    EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &test_data));
+    return test_data.AppendASCII("safe_browsing")
+        .AppendASCII("dmg")
+        .AppendASCII("data")
         .AppendASCII(file_name);
   }
 

chrome/browser/safe_browsing/download_protection/download_protection_service_unittest.cc

@@ -1467,9 +1467,10 @@ TEST_F(DownloadProtectionServiceTest,
                   net::URLRequestStatus::SUCCESS);
 
   base::FilePath unsigned_dmg;
-  EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &unsigned_dmg));
-  unsigned_dmg = unsigned_dmg.AppendASCII("chrome")
-                     .AppendASCII("safe_browsing_dmg")
+  EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &unsigned_dmg));
+  unsigned_dmg = unsigned_dmg.AppendASCII("safe_browsing")
+                     .AppendASCII("dmg")
+                     .AppendASCII("data")
                      .AppendASCII("mach_o_in_dmg.dmg");
 
   NiceMockDownloadItem item;
@@ -1503,9 +1504,10 @@ TEST_F(DownloadProtectionServiceTest,
                   net::URLRequestStatus::SUCCESS);
 
   base::FilePath test_data;
-  EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &test_data));
-  test_data = test_data.AppendASCII("chrome")
-                  .AppendASCII("safe_browsing_dmg")
+  EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &test_data));
+  test_data = test_data.AppendASCII("safe_browsing")
+                  .AppendASCII("dmg")
+                  .AppendASCII("data")
                   .AppendASCII("mach_o_in_dmg.txt");
 
   NiceMockDownloadItem item;
@@ -1536,9 +1538,10 @@ TEST_F(DownloadProtectionServiceTest, CheckClientDownloadReportDmgWithoutKoly) {
                   net::URLRequestStatus::SUCCESS);
 
   base::FilePath test_data;
-  EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &test_data));
-  test_data = test_data.AppendASCII("chrome")
-                  .AppendASCII("safe_browsing_dmg")
+  EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &test_data));
+  test_data = test_data.AppendASCII("safe_browsing")
+                  .AppendASCII("dmg")
+                  .AppendASCII("data")
                   .AppendASCII("mach_o_in_dmg_no_koly_signature.txt");
 
   NiceMockDownloadItem item;
@@ -1569,9 +1572,10 @@ TEST_F(DownloadProtectionServiceTest, CheckClientDownloadReportLargeDmg) {
                   net::URLRequestStatus::SUCCESS);
 
   base::FilePath unsigned_dmg;
-  EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &unsigned_dmg));
-  unsigned_dmg = unsigned_dmg.AppendASCII("chrome")
-                     .AppendASCII("safe_browsing_dmg")
+  EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &unsigned_dmg));
+  unsigned_dmg = unsigned_dmg.AppendASCII("safe_browsing")
+                     .AppendASCII("dmg")
+                     .AppendASCII("data")
                      .AppendASCII("mach_o_in_dmg.dmg");
 
   NiceMockDownloadItem item;
@@ -1606,9 +1610,10 @@ TEST_F(DownloadProtectionServiceTest, DMGAnalysisEndToEnd) {
                   net::URLRequestStatus::SUCCESS);
 
   base::FilePath dmg;
-  EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &dmg));
-  dmg = dmg.AppendASCII("chrome")
-            .AppendASCII("safe_browsing_dmg")
+  EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &dmg));
+  dmg = dmg.AppendASCII("safe_browsing")
+            .AppendASCII("dmg")
+            .AppendASCII("data")
             .AppendASCII("mach_o_in_dmg.dmg");
 
   NiceMockDownloadItem item;

chrome/services/file_util/public/cpp/sandboxed_dmg_analyzer_mac_unittest.cc

@@ -45,9 +45,10 @@ class SandboxedDMGAnalyzerTest : public testing::Test {
 
   base::FilePath GetFilePath(const char* file_name) {
     base::FilePath test_data;
-    EXPECT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &test_data));
-    return test_data.AppendASCII("chrome")
-        .AppendASCII("safe_browsing_dmg")
+    EXPECT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &test_data));
+    return test_data.AppendASCII("safe_browsing")
+        .AppendASCII("dmg")
+        .AppendASCII("data")
         .AppendASCII(file_name);
   }
 

chrome/test/BUILD.gn

@@ -3947,11 +3947,6 @@ test("unit_tests") {
       "//components/safe_browsing/db:v4_test_util",
       "//components/safe_browsing/renderer:websocket_sb_handshake_throttle_unittest",
     ]
-
-    if (is_mac) {
-      deps += [ ":mac_safe_browsing_test_data" ]
-      data += [ "$root_out_dir/test_data/chrome/safe_browsing_dmg/" ]
-    }
   }
 
   if (enable_plugins) {
@@ -5838,54 +5833,3 @@ if (!is_android && !is_fuchsia) {
     ]
   }
 }
-
-if (safe_browsing_mode == 1 && is_mac) {
-  action("mac_safe_browsing_test_data") {
-    script = "//build/gn_run_binary.py"
-    shell_script = "//chrome/test/data/safe_browsing/dmg/generate_test_data.sh"
-    inputs = [
-      script,
-      shell_script,
-      "data/safe_browsing/dmg/hfs_raw_images.tar.bz2",
-      "data/safe_browsing/mach_o/executablefat",
-      "data/safe_browsing/mach_o/lib64.dylib",
-    ]
-    _output_dir = "$root_out_dir/test_data/chrome/safe_browsing_dmg"
-    outputs = [
-      "$_output_dir/dmg_UDBZ_GPTSPUD.dmg",
-      "$_output_dir/dmg_UDBZ_NONE.dmg",
-      "$_output_dir/dmg_UDBZ_SPUD.dmg",
-      "$_output_dir/dmg_UDCO_GPTSPUD.dmg",
-      "$_output_dir/dmg_UDCO_NONE.dmg",
-      "$_output_dir/dmg_UDCO_SPUD.dmg",
-      "$_output_dir/dmg_UDRO_GPTSPUD.dmg",
-      "$_output_dir/dmg_UDRO_NONE.dmg",
-      "$_output_dir/dmg_UDRO_SPUD.dmg",
-      "$_output_dir/dmg_UDRW_GPTSPUD.dmg",
-      "$_output_dir/dmg_UDRW_NONE.dmg",
-      "$_output_dir/dmg_UDRW_SPUD.dmg",
-      "$_output_dir/dmg_UDSP_GPTSPUD.sparseimage",
-      "$_output_dir/dmg_UDSP_NONE.sparseimage",
-      "$_output_dir/dmg_UDSP_SPUD.sparseimage",
-      "$_output_dir/dmg_UDTO_GPTSPUD.cdr",
-      "$_output_dir/dmg_UDTO_NONE.cdr",
-      "$_output_dir/dmg_UDTO_SPUD.cdr",
-      "$_output_dir/dmg_UDZO_GPTSPUD.dmg",
-      "$_output_dir/dmg_UDZO_NONE.dmg",
-      "$_output_dir/dmg_UDZO_SPUD.dmg",
-      "$_output_dir/dmg_UFBI_GPTSPUD.dmg",
-      "$_output_dir/dmg_UFBI_NONE.dmg",
-      "$_output_dir/dmg_UFBI_SPUD.dmg",
-      "$_output_dir/hfs_plus.img",
-      "$_output_dir/hfsx_case_sensitive.img",
-      "$_output_dir/mach_o_in_dmg.dmg",
-      "$_output_dir/mach_o_in_dmg.txt",
-      "$_output_dir/mach_o_in_dmg_no_koly_signature.dmg",
-      "$_output_dir/mach_o_in_dmg_no_koly_signature.txt",
-    ]
-    args = [
-      rebase_path(shell_script, root_out_dir),
-      rebase_path(_output_dir, root_out_dir),
-    ]
-  }
-}

chrome/test/data/safe_browsing/dmg/.gitignore

-hfs_plus.img
-hfsx_case_sensitive.img
+data/

chrome/test/data/safe_browsing/dmg/Makefile

-# Copyright 2017 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-hfs_raw_images.tar.bz2: hfs_plus.img hfsx_case_sensitive.img
-	tar cjf $@ $^
-
-MAKE_HFS=./make_hfs.sh
-
-hfs_plus.img: $(MAKE_HFS)
-	rm -f $@
-	$(MAKE_HFS) HFS+ 1024 $@
-
-hfsx_case_sensitive.img: $(MAKE_HFS)
-	rm -f $@
-	# 8*1024 = 8192
-	$(MAKE_HFS) hfsx 8192 $@

chrome/test/data/safe_browsing/dmg/README.md

 # Safe Browsing DMG Test Data
 
 This directory contains scripts to generate test DMG and HFS files for
-unit-testing the Safe Browsing archive scanner. The contents of this directory
-are primarily used by `//chrome/test:mac_safe_browsing_test_data`.
+unit-testing the Safe Browsing archive scanner.
 
-Most of the data are generated at build-time using the `generate_test_data.sh`
-script. However, due to a [macOS issue](https://crbug.com/696529) the outputs
-from the `make_hfs.sh` script are generated independently and checked-in.
+The `generate_test_data.sh` script that produces the files in the `data/`
+subdirectory. It should be invoked like so:
 
-These independently generated data are stored in `hfs_raw_images.tar.bz2` and
-can be regenerated using the `Makefile` in this directory. The build system
-will extract the archive into the build output directory.
+    chrome/test/data/safe_browsing/dmg/generate_test_data.sh chrome/test/data/safe_browsing/dmg/data
+
+The script will produce the data files and bundle them for uploading to
+[CIPD](../../../../../docs/cipd.md). The script will produce a CIPD package
+named `data.zip`. The ZIP contents should be inspected and, if they look good,
+uploaded to the CIPD service and tagged with a version:
+
+    cipd pkg-register data.zip -tag version:YYYYMMDD.N
+
+For versioning the CIPD package, use a date in the form YYYYMMDD.N, where N is
+an integer starting at 1 to differentiate different versions on the same day.
+
+After uploading a new version to CIPD, the data need to be rolled into Chromium
+by updating the version referenced in the root `DEPS` file.
+
+Generating the data at build time is slow and has [caused](https://crbug.com/696529)
+[issues](https://crbug.com/817663) on the bots in the past.

chrome/test/data/safe_browsing/dmg/cipd.yaml

+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+package: chromium/chrome/test/data/safe_browsing/dmg
+description: Large-file test data for the Safe Browsing DMG/HFS tests on Mac.
+# Because the data are isolated for Swarming, which preserves symbolic links,
+# the data need to be copied.
+install_mode: copy
+data:
+  - dir: data

chrome/test/data/safe_browsing/dmg/generate_test_data.sh

@@ -30,9 +30,9 @@ generate_test_data() {
 
   # HFS Raw Images #############################################################
 
-  # Extract the checked-in testdata to the OUT_DIR, ignoring the archived
-  # modification times.
-  tar x -m -C "${OUT_DIR}" -f "${THIS_DIR}/hfs_raw_images.tar.bz2"
+  MAKE_HFS="${THIS_DIR}/make_hfs.sh"
+  "${MAKE_HFS}" HFS+ 1024 "${OUT_DIR}/hfs_plus.img"
+  "${MAKE_HFS}" hfsx $((8 * 1024)) "${OUT_DIR}/hfsx_case_sensitive.img"
 
   # DMG Files ##################################################################
 
@@ -103,6 +103,10 @@ generate_test_data() {
   # Copy of Mach-O DMG with extension changed to .txt and no 'koly' signature ##
   cp "${OUT_DIR}/mach_o_in_dmg_no_koly_signature.dmg" \
       "${OUT_DIR}/mach_o_in_dmg_no_koly_signature.txt"
+
+  # Package Data for CIPD ######################################################
+
+  cipd pkg-build -pkg-def "${THIS_DIR}/cipd.yaml" -out "${THIS_DIR}/data.zip"
 }
 
 # Silence any stdout, but keep stderr.

chrome/utility/safe_browsing/mac/dmg_test_utils.cc

@@ -18,10 +18,11 @@ namespace test {
 
 void GetTestFile(const char* file_name, base::File* file) {
   base::FilePath test_data;
-  ASSERT_TRUE(base::PathService::Get(chrome::DIR_GEN_TEST_DATA, &test_data));
+  ASSERT_TRUE(base::PathService::Get(chrome::DIR_TEST_DATA, &test_data));
 
-  base::FilePath path = test_data.AppendASCII("chrome")
-      .AppendASCII("safe_browsing_dmg")
+  base::FilePath path = test_data.AppendASCII("safe_browsing")
+                            .AppendASCII("dmg")
+                            .AppendASCII("data")
                             .AppendASCII(file_name);
 
   *file = base::File(path, base::File::FLAG_OPEN | base::File::FLAG_READ);