DNR: Add filter_list_converter tool.

Filter List Converter is a tool to convert filter list files in the text format
to a JSON file in a format supported by the Declarative Net Request API. It can
either output the complete extension (with a manifest file and a JSON ruleset)
or just the JSON ruleset.

This is based on the ruleset converter module in the subresource_filter
component.

BUG=972290

Change-Id: Id67ccaa5cc3cc469da67865c3754b41182d1ab9f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1139592
Auto-Submit: Karan Bhatia <karandeepb@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Istiaque Ahmed <lazyboy@chromium.org>
Reviewed-by: Charlie Harrison <csharrison@chromium.org>
Commit-Queue: Karan Bhatia <karandeepb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#672233}

BUILD.gn

@@ -230,6 +230,7 @@ group("gn_all") {
     deps += [
       "//extensions:extensions_browsertests",
       "//extensions:extensions_unittests",
+      "//extensions/browser/api/declarative_net_request/filter_list_converter",
       "//extensions/shell:app_shell_unittests",
     ]
   }

extensions/browser/BUILD.gn

@@ -686,6 +686,7 @@ source_set("unit_tests") {
     "//device/bluetooth:mocks",
     "//extensions:extensions_browser_resources",
     "//extensions:test_support",
+    "//extensions/browser/api/declarative_net_request/filter_list_converter:unit_tests",
     "//extensions/buildflags",
     "//extensions/common",
     "//extensions/common:test_support",

extensions/browser/api/declarative_net_request/filter_list_converter/BUILD.gn

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//extensions/buildflags/buildflags.gni")
+
+assert(enable_extensions)
+
+source_set("support") {
+  testonly = true
+  sources = [
+    "converter.cc",
+    "converter.h",
+  ]
+  deps = [
+    "//base",
+    "//components/subresource_filter/tools/ruleset_converter:support",
+    "//extensions/browser",
+    "//extensions/common",
+    "//extensions/common:test_support",
+    "//url",
+  ]
+}
+
+executable("filter_list_converter") {
+  testonly = true
+  sources = [
+    "main.cc",
+  ]
+  deps = [
+    ":support",
+    "//base",
+  ]
+}
+
+source_set("unit_tests") {
+  testonly = true
+  sources = [
+    "converter_unittest.cc",
+  ]
+  deps = [
+    ":support",
+    "//base",
+    "//testing/gtest",
+  ]
+}

extensions/browser/api/declarative_net_request/filter_list_converter/DEPS

+include_rules = [
+  "+components/subresource_filter/tools/ruleset_converter",
+]

extensions/browser/api/declarative_net_request/filter_list_converter/README

+Filter List Converter is a tool to convert filter list files in the text format
+to a JSON file in a format supported by the Declarative Net Request API. It can
+either output the complete extension or just the JSON ruleset.
+
+This is based on the ruleset converter module in the subresource_filter
+component.

extensions/browser/api/declarative_net_request/filter_list_converter/converter.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef EXTENSIONS_BROWSER_API_DECLARATIVE_NET_REQUEST_FILTER_LIST_CONVERTER_CONVERTER_H_
+#define EXTENSIONS_BROWSER_API_DECLARATIVE_NET_REQUEST_FILTER_LIST_CONVERTER_CONVERTER_H_
+
+#include <vector>
+
+#include "base/files/file_path.h"
+#include "base/macros.h"
+
+namespace extensions {
+namespace declarative_net_request {
+namespace filter_list_converter {
+
+enum WriteType {
+  kJSONRuleset,
+  kExtension,
+};
+
+// Utility function to convert filter list files in the text format to a JSON
+// file in a format supported by the Declarative Net Request API. If |type| is
+// kExtension, output_path is treated as the extension directory and the ruleset
+// is written to "rules.json". Else it is treated as the json ruleset location.
+// Returns false if the conversion fails.
+bool ConvertRuleset(const std::vector<base::FilePath>& filter_list_inputs,
+                    const base::FilePath& output_path,
+                    WriteType type,
+                    bool noisy = true);
+
+}  // namespace filter_list_converter
+}  // namespace declarative_net_request
+}  // namespace extensions
+
+#endif  // EXTENSIONS_BROWSER_API_DECLARATIVE_NET_REQUEST_FILTER_LIST_CONVERTER_CONVERTER_H_

extensions/browser/api/declarative_net_request/filter_list_converter/converter_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "extensions/browser/api/declarative_net_request/filter_list_converter/converter.h"
+
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/files/scoped_temp_dir.h"
+#include "base/json/json_file_value_serializer.h"
+#include "base/json/json_reader.h"
+#include "base/logging.h"
+#include "base/optional.h"
+#include "base/strings/string_util.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace extensions {
+namespace declarative_net_request {
+namespace filter_list_converter {
+namespace {
+
+void TestConversion(std::vector<std::string> filter_list_rules,
+                    std::string json_result,
+                    WriteType write_type) {
+  base::ScopedTempDir temp_dir;
+  CHECK(temp_dir.CreateUniqueTempDir());
+
+  base::FilePath input_path = temp_dir.GetPath().AppendASCII("filterlist.txt");
+  std::string filterlist = base::JoinString(filter_list_rules, "\n");
+  CHECK_EQ(filterlist.size(),
+           static_cast<size_t>(base::WriteFile(input_path, filterlist.c_str(),
+                                               filterlist.size())));
+
+  base::Optional<base::Value> expected_json =
+      base::JSONReader::Read(json_result);
+  CHECK(expected_json.has_value());
+
+  base::FilePath output_path = temp_dir.GetPath();
+  if (write_type == WriteType::kJSONRuleset)
+    output_path = output_path.AppendASCII("rules.json");
+
+  ConvertRuleset({input_path}, output_path, write_type, false /* noisy */);
+
+  base::FilePath output_json_path =
+      temp_dir.GetPath().AppendASCII("rules.json");
+  JSONFileValueDeserializer deserializer(output_json_path);
+  std::unique_ptr<base::Value> actual_json = deserializer.Deserialize(
+      nullptr /* error_code */, nullptr /* error_message */);
+  ASSERT_TRUE(actual_json.get());
+
+  EXPECT_EQ(*expected_json, *actual_json);
+
+  if (write_type == WriteType::kExtension) {
+    EXPECT_TRUE(
+        base::PathExists(temp_dir.GetPath().AppendASCII("manifest.json")));
+  }
+}
+
+class FilterListConverterTest : public ::testing::TestWithParam<WriteType> {};
+
+TEST_P(FilterListConverterTest, Convert) {
+  std::vector<std::string> filter_list_rules = {
+      "||example.com^|$script,image,font",
+      "@@allowed.com$domain=example.com|~sub.example.com",
+      "|https://*.abc.com|$match-case,~image,third-party",
+      "abc.com$~third-party"};
+
+  std::string expected_result = R"(
+    [ {
+       "action": {
+          "type": "block"
+       },
+       "condition": {
+          "isUrlFilterCaseSensitive": false,
+          "resourceTypes": [ "script", "image", "font" ],
+          "urlFilter": "||example.com^|"
+       },
+       "id": 1
+    }, {
+       "action": {
+          "type": "allow"
+       },
+       "condition": {
+          "domains": [ "example.com" ],
+          "excludedDomains": [ "sub.example.com" ],
+          "isUrlFilterCaseSensitive": false,
+          "urlFilter": "allowed.com"
+       },
+       "id": 2
+    }, {
+       "action": {
+          "type": "block"
+       },
+       "condition": {
+          "resourceTypes": [ "other", "script", "stylesheet", "object",
+              "xmlhttprequest", "sub_frame", "ping", "media", "font",
+              "websocket" ],
+          "urlFilter": "|https://*.abc.com|",
+          "domainType": "thirdParty"
+       },
+       "id": 3
+    }, {
+       "action": {
+          "type": "block"
+       },
+       "condition": {
+          "isUrlFilterCaseSensitive": false,
+          "urlFilter": "abc.com",
+          "domainType": "firstParty"
+       },
+       "id": 4
+    } ]
+
+)";
+
+  TestConversion(filter_list_rules, expected_result, GetParam());
+}
+
+INSTANTIATE_TEST_SUITE_P(,
+                         FilterListConverterTest,
+                         ::testing::Values(WriteType::kExtension,
+                                           WriteType::kJSONRuleset));
+
+}  // namespace
+}  // namespace filter_list_converter
+}  // namespace declarative_net_request
+}  // namespace extensions

extensions/browser/api/declarative_net_request/filter_list_converter/main.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <vector>
+
+#include "base/command_line.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/logging.h"
+#include "base/strings/string_split.h"
+#include "base/strings/stringprintf.h"
+#include "extensions/browser/api/declarative_net_request/filter_list_converter/converter.h"
+
+namespace {
+
+const char kSwitchInputFilterlistFiles[] = "input_filterlists";
+const char kSwitchOutputPath[] = "output_path";
+const char kSwitchOutputType[] = "output_type";
+const char kOutputTypeExtension[] = "extension";
+const char kOutputTypeJSON[] = "json";
+const char kJSONExtension[] = ".json";
+
+const char kHelpMsg[] = R"(
+  filter_list_converter --input_filterlists=[<path1>, <path2>]
+          --output_path=<path> --output_type=<extension,json>
+
+  Filter List Converter is a tool to convert filter list files in the text
+  format to a JSON file in a format supported by the Declarative Net Request
+  API. It can either output the complete extension or just the JSON ruleset.
+
+  --input_filterlists = List of input paths to text filter list files.
+  --output_path = The output path. The parent directory should exist.
+  --output_type = Optional switch. One of "extension" or "json". "json" is the
+                  default.
+)";
+
+namespace filter_list_converter =
+    extensions::declarative_net_request::filter_list_converter;
+
+void PrintHelp() {
+  LOG(ERROR) << kHelpMsg;
+}
+
+}  // namespace
+
+int main(int argc, char* argv[]) {
+  base::CommandLine::Init(argc, argv);
+  base::CommandLine& command_line = *base::CommandLine::ForCurrentProcess();
+
+  if (!command_line.HasSwitch(kSwitchInputFilterlistFiles) ||
+      !command_line.HasSwitch(kSwitchOutputPath)) {
+    PrintHelp();
+    return 1;
+  }
+
+  std::vector<base::FilePath> input_paths;
+  base::CommandLine::StringType comma_separated_paths =
+      command_line.GetSwitchValueNative(kSwitchInputFilterlistFiles);
+  for (const auto& piece : base::SplitStringPiece(
+           comma_separated_paths, "," /* separator */, base::TRIM_WHITESPACE,
+           base::SPLIT_WANT_NONEMPTY)) {
+    base::FilePath path(piece);
+
+    if (!base::PathExists(path)) {
+      LOG(ERROR) << "Input path " << piece << " does not exist.";
+      return 1;
+    }
+
+    input_paths.push_back(path);
+  }
+  if (input_paths.empty()) {
+    LOG(ERROR) << base::StringPrintf(
+        "No valid input files specified using '%s'.",
+        kSwitchInputFilterlistFiles);
+    return 1;
+  }
+
+  filter_list_converter::WriteType write_type =
+      filter_list_converter::kJSONRuleset;
+  if (command_line.HasSwitch(kSwitchOutputType)) {
+    std::string output_type =
+        command_line.GetSwitchValueASCII(kSwitchOutputType);
+    if (output_type == kOutputTypeExtension) {
+      write_type = filter_list_converter::kExtension;
+    } else if (output_type == kOutputTypeJSON) {
+      write_type = filter_list_converter::kJSONRuleset;
+    } else {
+      LOG(ERROR) << base::StringPrintf("Invalid value for switch '%s'",
+                                       kSwitchOutputType);
+      return 1;
+    }
+  }
+
+  base::FilePath output_path =
+      command_line.GetSwitchValuePath(kSwitchOutputPath);
+  bool invalid_output_path = false;
+  switch (write_type) {
+    case filter_list_converter::kExtension:
+      invalid_output_path = !base::DirectoryExists(output_path);
+      break;
+    case filter_list_converter::kJSONRuleset:
+      invalid_output_path = output_path.Extension() != kJSONExtension;
+      invalid_output_path |= !base::DirectoryExists(output_path.DirName());
+      break;
+  }
+  if (invalid_output_path) {
+    LOG(ERROR) << "Invalid output path " << output_path.value();
+    return 1;
+  }
+
+  if (!filter_list_converter::ConvertRuleset(input_paths, output_path,
+                                             write_type)) {
+    LOG(ERROR) << "Conversion failed.";
+    return 1;
+  }
+
+  return 0;
+}

extensions/browser/api/declarative_net_request/indexed_rule.cc

@@ -272,7 +272,6 @@ ParseResult IndexedRule::CreateIndexedRule(dnr_api::Rule parsed_rule,
                                            const GURL& base_url,
                                            IndexedRule* indexed_rule) {
   DCHECK(indexed_rule);
-  DCHECK(IsAPIAvailable());
 
   if (parsed_rule.id < kMinValidID)
     return ParseResult::ERROR_INVALID_RULE_ID;

extensions/browser/api/declarative_net_request/ruleset_source.cc

@@ -367,6 +367,7 @@ ParseInfo RulesetSource::IndexAndPersistRules(std::vector<dnr_api::Rule> rules,
                                               int* ruleset_checksum) const {
   DCHECK_LE(rules.size(), rule_count_limit_);
   DCHECK(ruleset_checksum);
+  DCHECK(IsAPIAvailable());
 
   FlatRulesetIndexer indexer;