Add ruleset_converter tool to subresource_filter

This CL also
- Aggregates all subresource_filter tools into a single target
- changes subresource_filter tools to use the same (but equiv)
  condition as some other tools
- Adds that target to "all"

Bug: 833419
Change-Id: I732ac0860f90e036281bdf899f8b5e6e92f3f3e5
Reviewed-on: https://chromium-review.googlesource.com/1031211Reviewed-by: Scott Violet <sky@chromium.org>
Reviewed-by: Josh Karlin <jkarlin@chromium.org>
Commit-Queue: Charlie Harrison <csharrison@chromium.org>
Cr-Commit-Position: refs/heads/master@{#554455}

BUILD.gn

@@ -101,6 +101,7 @@ group("gn_all") {
       "//chrome/test:interactive_ui_tests",
       "//chrome/test:sync_integration_tests",
       "//chrome/test/chromedriver:chromedriver_unittests",
+      "//components/subresource_filter/tools:subresource_filter_tools",
       "//components/sync/tools:sync_client",
       "//components/sync/tools:sync_listen_notifications",
       "//components/zucchini:zucchini",

components/subresource_filter/tools/BUILD.gn

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import("//build/config/chromecast_build.gni")
+
 static_library("tools_lib") {
   sources = [
     "filter_tool.cc",
@@ -38,9 +40,10 @@ source_set("unit_tests") {
     "//url",
   ]
 }
-
-if (is_linux || is_mac || is_win) {
+if (!is_ios && !is_android && !is_chromecast && !is_fuchsia) {
   executable("subresource_filter_tool") {
+    # Production code should not depend on this.
+    testonly = true
     sources = [
       "filter_tool_main.cc",
     ]
@@ -52,6 +55,8 @@ if (is_linux || is_mac || is_win) {
   }
 
   executable("subresource_indexing_tool") {
+    # Production code should not depend on this.
+    testonly = true
     sources = [
       "indexing_tool_main.cc",
     ]
@@ -61,4 +66,14 @@ if (is_linux || is_mac || is_win) {
       "//build/config:exe_and_shlib_deps",
     ]
   }
+
+  group("subresource_filter_tools") {
+    # Production code should not depend on this.
+    testonly = true
+    deps = [
+      ":subresource_filter_tool",
+      ":subresource_indexing_tool",
+      "ruleset_converter",
+    ]
+  }
 }

components/subresource_filter/tools/ruleset_converter/BUILD.gn

@@ -2,7 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-source_set("ruleset_converter_support") {
+import("//build/config/chromecast_build.gni")
+
+source_set("support") {
   sources = [
     "//third_party/protobuf:protobuf_lite",
     "rule_stream.cc",
@@ -25,7 +27,7 @@ source_set("unit_tests") {
     "rule_stream_test.cc",
   ]
   deps = [
-    ":ruleset_converter_support",
+    ":support",
     "../rule_parser",
     "//base",
     "//base/test:test_support",
@@ -35,3 +37,17 @@ source_set("unit_tests") {
     "//testing/gtest",
   ]
 }
+
+if (!is_ios && !is_android && !is_chromecast && !is_fuchsia) {
+  executable("ruleset_converter") {
+    sources = [
+      "main.cc",
+    ]
+    deps = [
+      ":support",
+      "//base",
+      "//build/config:exe_and_shlib_deps",
+      "//third_party/protobuf:protobuf_lite",
+    ]
+  }
+}

components/subresource_filter/tools/ruleset_converter/main.cc

+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <cstdio>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "base/command_line.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/logging.h"
+#include "base/strings/string_number_conversions.h"
+#include "base/strings/string_piece.h"
+#include "base/strings/string_split.h"
+#include "components/subresource_filter/tools/ruleset_converter/rule_stream.h"
+#include "components/subresource_filter/tools/ruleset_converter/ruleset_format.h"
+
+namespace {
+
+const char kSwitchInputFiles[] = "input_files";
+const char kSwitchOutputFile[] = "output_file";
+
+const char kSwitchOutputFileUrl[] = "output_file_url";
+const char kSwitchOutputFileCss[] = "output_file_css";
+
+const char kSwitchInputFormat[] = "input_format";
+const char kSwitchOutputFormat[] = "output_format";
+
+const char kSwitchChromeVersion[] = "chrome_version";
+
+const char kHelpMsg[] = R"(
+  ruleset_converter [--input_format=<format>] --output_format=<format>
+  --input_files=<path1>[:<path2>...]
+  (--output_file=<path> | [--output_file_url=<path>] [--output_file_css=<path>)
+  [--chrome_version=<version>]
+
+  ruleset_converter is a utility for converting subresource_filter rulesets
+  across multiple formats:
+
+  * --input_files: Colon-separated list of input files with rules. The files
+     are processed in the order of declaration
+
+  * --output_file: The file to output the rules. Either this option or at least
+    one of the --output_file_url|--output_file_css should be specified.
+
+  * --output_file_url: The file to output URL rules. If equal to
+    --output_file_css, the results are merged.
+
+  *  --output_file_css: The file to output CSS rules. See --output_file and
+     --output_file_url for details.
+
+  * --input_format: The format of the input file(s). One of
+    {filter-list, proto, unindexed-ruleset}
+
+  * --output_format: The format of the output file(s). See --input_format for
+    available formats.
+
+  * --chrome_version: The earliest version of Chrome that the produced ruleset
+    needs to be compatible with. Currently one of 54, 59, or 0
+    (not Chrome-specific). Defaults to the maximum (i.e. 59).
+)";
+
+void PrintHelp() {
+  printf("%s\n\n", kHelpMsg);
+}
+
+}  // namespace
+
+int main(int argc, char* argv[]) {
+  base::CommandLine::Init(argc, argv);
+  base::CommandLine& command_line = *base::CommandLine::ForCurrentProcess();
+
+  if (command_line.GetArgs().size() != 0U) {
+    PrintHelp();
+    return 1;
+  }
+
+  if (!command_line.HasSwitch(kSwitchInputFiles)) {
+    std::fprintf(stderr, "--input_files flag is not specified.\n");
+    PrintHelp();
+    return 1;
+  }
+
+  if (command_line.GetSwitchValueASCII(kSwitchOutputFile).empty() &&
+      command_line.GetSwitchValueASCII(kSwitchOutputFileUrl).empty() &&
+      command_line.GetSwitchValueASCII(kSwitchOutputFileCss).empty()) {
+    std::fprintf(stderr,
+                 "Either --output_file, or at least of one "
+                 "--output_file_url|--output_file_css should be specified.\n");
+    PrintHelp();
+    return 1;
+  }
+
+  const subresource_filter::RulesetFormat input_format =
+      command_line.HasSwitch(kSwitchInputFormat)
+          ? subresource_filter::ParseFlag(
+                command_line.GetSwitchValueASCII(kSwitchInputFormat))
+          : subresource_filter::RulesetFormat::kFilterList;
+  if (input_format == subresource_filter::RulesetFormat::kUndefined) {
+    std::fprintf(stderr, "Input format is not defined.\n");
+    return 1;
+  }
+
+  const subresource_filter::RulesetFormat output_format =
+      subresource_filter::ParseFlag(
+          command_line.GetSwitchValueASCII(kSwitchOutputFormat));
+  if (output_format == subresource_filter::RulesetFormat::kUndefined) {
+    std::fprintf(stderr, "Output format is not defined.\n");
+    return 1;
+  }
+
+  int chrome_version = 59;
+  if (command_line.HasSwitch(kSwitchChromeVersion)) {
+    if (!base::StringToInt(
+            command_line.GetSwitchValueASCII(kSwitchChromeVersion),
+            &chrome_version)) {
+      fprintf(stderr, "Unable to parse chrome version");
+      return 1;
+    }
+  }
+  if (chrome_version != 0 && chrome_version != 54 && chrome_version != 59) {
+    std::fprintf(stderr, "chrome_version should be in {0, 54, 59}.\n");
+    return 1;
+  }
+
+  // Vet the input paths.
+  auto inputs = command_line.GetSwitchValueNative(kSwitchInputFiles);
+  std::vector<base::FilePath> input_paths;
+  for (const auto& piece : base::SplitStringPieceUsingSubstr(
+           inputs, ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_NONEMPTY)) {
+    base::FilePath path(piece);
+    if (!base::PathExists(path)) {
+      std::fprintf(stderr, "Input path does not exist\n");
+      return 1;
+    }
+    input_paths.push_back(path);
+  }
+
+  // Create output stream(s).
+  std::unique_ptr<subresource_filter::RuleOutputStream> primary_output;
+  std::unique_ptr<subresource_filter::RuleOutputStream> secondary_output;
+  subresource_filter::RuleOutputStream* css_rules_output = nullptr;
+
+  base::FilePath primary_filename =
+      command_line.GetSwitchValuePath(kSwitchOutputFile);
+  const bool single_output = !primary_filename.empty();
+  if (!single_output)
+    primary_filename = command_line.GetSwitchValuePath(kSwitchOutputFileUrl);
+
+  if (!primary_filename.empty()) {
+    if (!base::DirectoryExists(primary_filename.DirName())) {
+      std::fprintf(stderr, "Output directory does not exist\n");
+      return 1;
+    }
+    primary_output = subresource_filter::RuleOutputStream::Create(
+        std::make_unique<std::ofstream>(primary_filename.AsUTF8Unsafe(),
+                                        std::ios::binary | std::ios::out),
+        output_format);
+  }
+
+  base::FilePath secondary_filename =
+      command_line.GetSwitchValuePath(kSwitchOutputFileCss);
+  if (single_output || secondary_filename == primary_filename) {
+    css_rules_output = primary_output.get();
+  } else if (!secondary_filename.empty()) {
+    if (!base::DirectoryExists(secondary_filename.DirName())) {
+      std::fprintf(stderr, "Output directory does not exist\n");
+      return 1;
+    }
+    secondary_output = subresource_filter::RuleOutputStream::Create(
+        std::make_unique<std::ofstream>(secondary_filename.AsUTF8Unsafe(),
+                                        std::ios::binary | std::ios::out),
+        output_format);
+    css_rules_output = secondary_output.get();
+  }
+
+  // Iterate through input files and stream them to the outputs.
+  for (const auto& path : input_paths) {
+    auto input_stream = subresource_filter::RuleInputStream::Create(
+        std::make_unique<std::ifstream>(path.AsUTF8Unsafe(),
+                                        std::ios::binary | std::ios::in),
+        input_format);
+    CHECK(input_stream);
+    CHECK(subresource_filter::TransferRules(input_stream.get(),
+                                            primary_output.get(),
+                                            css_rules_output, chrome_version));
+  }
+  if (primary_output)
+    CHECK(primary_output->Finish());
+  if (secondary_output)
+    CHECK(secondary_output->Finish());
+  return 0;
+}

components/subresource_filter/tools/ruleset_converter/rule_stream.cc

@@ -298,6 +298,7 @@ std::unique_ptr<RuleInputStream> RuleInputStream::Create(
     std::unique_ptr<std::istream> input,
     RulesetFormat format) {
   CHECK(input);
+  CHECK(!input->bad());
   std::unique_ptr<RuleInputStream> result;
   switch (format) {
     case RulesetFormat::kFilterList:
@@ -321,6 +322,7 @@ std::unique_ptr<RuleOutputStream> RuleOutputStream::Create(
     std::unique_ptr<std::ostream> output,
     RulesetFormat format) {
   CHECK(output);
+  CHECK(!output->bad());
   std::unique_ptr<RuleOutputStream> result;
   switch (format) {
     case RulesetFormat::kFilterList:

components/subresource_filter/tools/ruleset_converter/ruleset_format.cc

@@ -8,42 +8,14 @@
 
 namespace subresource_filter {
 
-bool ParseFlag(const std::string& text,
-               RulesetFormat* format,
-               std::string* error) {
-  if (text.empty()) {
-    *format = RulesetFormat::kUndefined;
-    return true;
-  }
-  if (text == "filter-list") {
-    *format = RulesetFormat::kFilterList;
-    return true;
-  }
-  if (text == "proto") {
-    *format = RulesetFormat::kProto;
-    return true;
-  }
-  if (text == "unindexed-ruleset") {
-    *format = RulesetFormat::kUnindexedRuleset;
-    return true;
-  }
-  *error = "unknown RulesetFormat";
-  return false;
-}
-
-std::string UnparseFlag(RulesetFormat format) {
-  switch (format) {
-    case RulesetFormat::kUndefined:
-      return "";
-    case RulesetFormat::kFilterList:
-      return "filter-list";
-    case RulesetFormat::kProto:
-      return "proto";
-    case RulesetFormat::kUnindexedRuleset:
-      return "unindexed-ruleset";
-    default:
-      return base::IntToString(static_cast<int>(format));
-  }
+RulesetFormat ParseFlag(const std::string& text) {
+  if (text == "filter-list")
+    return RulesetFormat::kFilterList;
+  if (text == "proto")
+    return RulesetFormat::kProto;
+  if (text == "unindexed-ruleset")
+    return RulesetFormat::kUnindexedRuleset;
+  return RulesetFormat::kUndefined;
 }
 
 }  // namespace subresource_filter

components/subresource_filter/tools/ruleset_converter/ruleset_format.h

@@ -24,11 +24,7 @@ enum class RulesetFormat {
   kUnindexedRuleset,
 };
 
-// Functions used for compatibility with "base/commandlineflags.h".
-bool ParseFlag(const std::string& text,
-               RulesetFormat* format,
-               std::string* error);
-std::string UnparseFlag(RulesetFormat format);
+RulesetFormat ParseFlag(const std::string& text);
 
 }  // namespace subresource_filter