DNR: Add filter_list_converter tool.

Filter List Converter is a tool to convert filter list files in the text format
to a JSON file in a format supported by the Declarative Net Request API. It can
either output the complete extension (with a manifest file and a JSON ruleset)
or just the JSON ruleset.

This is based on the ruleset converter module in the subresource_filter
component.

BUG=972290

Change-Id: Id67ccaa5cc3cc469da67865c3754b41182d1ab9f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1139592
Auto-Submit: Karan Bhatia <karandeepb@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Istiaque Ahmed <lazyboy@chromium.org>
Reviewed-by: Charlie Harrison <csharrison@chromium.org>
Commit-Queue: Karan Bhatia <karandeepb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#672233}

BUILD.gn

@@ -230,6 +230,7 @@ group("gn_all") {
     deps += [
       "//extensions:extensions_browsertests",
       "//extensions:extensions_unittests",
+      "//extensions/browser/api/declarative_net_request/filter_list_converter",
       "//extensions/shell:app_shell_unittests",
     ]
   }

extensions/browser/BUILD.gn

@@ -686,6 +686,7 @@ source_set("unit_tests") {
     "//device/bluetooth:mocks",
     "//extensions:extensions_browser_resources",
     "//extensions:test_support",
+    "//extensions/browser/api/declarative_net_request/filter_list_converter:unit_tests",
     "//extensions/buildflags",
     "//extensions/common",
     "//extensions/common:test_support",

extensions/browser/api/declarative_net_request/filter_list_converter/BUILD.gn

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//extensions/buildflags/buildflags.gni")
+
+assert(enable_extensions)
+
+source_set("support") {
+  testonly = true
+  sources = [
+    "converter.cc",
+    "converter.h",
+  ]
+  deps = [
+    "//base",
+    "//components/subresource_filter/tools/ruleset_converter:support",
+    "//extensions/browser",
+    "//extensions/common",
+    "//extensions/common:test_support",
+    "//url",
+  ]
+}
+
+executable("filter_list_converter") {
+  testonly = true
+  sources = [
+    "main.cc",
+  ]
+  deps = [
+    ":support",
+    "//base",
+  ]
+}
+
+source_set("unit_tests") {
+  testonly = true
+  sources = [
+    "converter_unittest.cc",
+  ]
+  deps = [
+    ":support",
+    "//base",
+    "//testing/gtest",
+  ]
+}

extensions/browser/api/declarative_net_request/filter_list_converter/DEPS

+include_rules = [
+  "+components/subresource_filter/tools/ruleset_converter",
+]

extensions/browser/api/declarative_net_request/filter_list_converter/README

+Filter List Converter is a tool to convert filter list files in the text format
+to a JSON file in a format supported by the Declarative Net Request API. It can
+either output the complete extension or just the JSON ruleset.
+
+This is based on the ruleset converter module in the subresource_filter
+component.

extensions/browser/api/declarative_net_request/filter_list_converter/converter.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "extensions/browser/api/declarative_net_request/filter_list_converter/converter.h"
+
+#include <fstream>
+#include <string>
+#include <utility>
+
+#include "base/json/json_file_value_serializer.h"
+#include "base/logging.h"
+#include "base/strings/string_util.h"
+#include "base/strings/stringprintf.h"
+#include "base/values.h"
+#include "components/subresource_filter/tools/ruleset_converter/rule_stream.h"
+#include "extensions/browser/api/declarative_net_request/constants.h"
+#include "extensions/browser/api/declarative_net_request/indexed_rule.h"
+#include "extensions/common/api/declarative_net_request.h"
+#include "extensions/common/api/declarative_net_request/constants.h"
+#include "extensions/common/api/declarative_net_request/test_utils.h"
+#include "url/gurl.h"
+
+namespace extensions {
+namespace declarative_net_request {
+
+namespace {
+
+namespace proto = ::url_pattern_index::proto;
+namespace dnr_api = extensions::api::declarative_net_request;
+
+using ElementTypeMap =
+    base::flat_map<proto::ElementType, dnr_api::ResourceType>;
+
+constexpr char kJSONRulesFilename[] = "rules.json";
+const base::FilePath::CharType kJSONRulesetFilepath[] =
+    FILE_PATH_LITERAL("rules.json");
+
+// Utility class to convert the proto::UrlRule format to the JSON format
+// supported by Declarative Net Request.
+class ProtoToJSONRuleConverter {
+ public:
+  // Returns a dictionary value corresponding to a Declarative Net Request rule
+  // on success. On error, returns an empty/null value and populates |error|.
+  // |error| must be non-null.
+  static base::Value Convert(const proto::UrlRule& rule,
+                             int rule_id,
+                             std::string* error) {
+    CHECK(error);
+    ProtoToJSONRuleConverter json_rule(rule, rule_id);
+    return json_rule.Convert(error);
+  }
+
+ private:
+  ProtoToJSONRuleConverter(const proto::UrlRule& rule, int rule_id)
+      : input_rule_(rule),
+        rule_id_(rule_id),
+        json_rule_(base::Value::Type::DICTIONARY) {}
+
+  base::Value Convert(std::string* error) {
+    CHECK(error);
+
+    // Populate all the keys.
+    bool success = CheckActivationType() && PopulateID() &&
+                   PopulatePriorirty() && PopulateURLFilter() &&
+                   PopulateIsURLFilterCaseSensitive() && PopulateDomains() &&
+                   PopulateExcludedDomains() && PopulateResourceTypes() &&
+                   PopulateExcludedResourceTypes() && PopulateDomainType() &&
+                   PopulateRuleActionType() && PopulateRedirectURL() &&
+                   PopulateRemoveHeadersList();
+
+    if (!success) {
+      CHECK(!error_.empty());
+      *error = std::move(error_);
+      return base::Value();
+    }
+
+    // Sanity check that we can parse this rule.
+    base::string16 err;
+    dnr_api::Rule rule;
+    CHECK(dnr_api::Rule::Populate(json_rule_, &rule, &err) && err.empty())
+        << "Converted rule can't be parsed " << json_rule_;
+
+    IndexedRule indexed_rule;
+    ParseResult result = IndexedRule::CreateIndexedRule(
+        std::move(rule), GURL() /* base_url */, &indexed_rule);
+
+    auto get_non_ascii_error = [this](const std::string& context) {
+      return base::StringPrintf(
+          "Rule with filter '%s' ignored due to non ascii characters in %s.",
+          input_rule_.url_pattern().c_str(), context.c_str());
+    };
+
+    // Non-ascii characters in rules are not supported.
+    if (result == ParseResult::ERROR_NON_ASCII_URL_FILTER) {
+      *error = get_non_ascii_error("url filter");
+      return base::Value();
+    }
+    if (result == ParseResult::ERROR_NON_ASCII_DOMAIN) {
+      *error = get_non_ascii_error("domains");
+      return base::Value();
+    }
+    if (result == ParseResult::ERROR_NON_ASCII_EXCLUDED_DOMAIN) {
+      *error = get_non_ascii_error("excluded domains");
+      return base::Value();
+    }
+
+    CHECK_EQ(ParseResult::SUCCESS, result)
+        << "Unexpected parse error << " << static_cast<int>(result)
+        << " for rule " << json_rule_;
+
+    return std::move(json_rule_);
+  }
+
+  bool CheckActivationType() {
+    if (input_rule_.activation_types() == proto::ACTIVATION_TYPE_UNSPECIFIED)
+      return true;
+
+    std::vector<std::string> activation_types;
+    for (int activation_type = 1; activation_type <= proto::ACTIVATION_TYPE_MAX;
+         activation_type <<= 1) {
+      CHECK(proto::ActivationType_IsValid(activation_type));
+      if (!(input_rule_.activation_types() & activation_type))
+        continue;
+
+      switch (static_cast<proto::ActivationType>(activation_type)) {
+        case proto::ACTIVATION_TYPE_UNSPECIFIED:
+          CHECK(false);
+          break;
+        case proto::ACTIVATION_TYPE_DOCUMENT:
+          activation_types.emplace_back("document");
+          break;
+        case proto::ACTIVATION_TYPE_ELEMHIDE:
+          activation_types.emplace_back("elemhide");
+          break;
+        case proto::ACTIVATION_TYPE_GENERICHIDE:
+          activation_types.emplace_back("generichide");
+          break;
+        case proto::ACTIVATION_TYPE_GENERICBLOCK:
+          activation_types.emplace_back("genericblock");
+          break;
+        case proto::ACTIVATION_TYPE_ALL:
+          CHECK(false);
+          break;
+      }
+    }
+
+    // We don't support any activation types.
+    error_ = base::StringPrintf(
+        "Rule with filter '%s' ignored due to invalid activation types-[%s].",
+        input_rule_.url_pattern().c_str(),
+        base::JoinString(activation_types, "," /* separator */).c_str());
+    return false;
+  }
+
+  bool PopulateID() {
+    CHECK_GE(rule_id_, kMinValidID);
+    CHECK(json_rule_.SetKey(kIDKey, base::Value(rule_id_)));
+    return true;
+  }
+
+  bool PopulatePriorirty() {
+    // Do nothing. Priority is optional and only relevant for redirect rules.
+    return true;
+  }
+
+  bool PopulateURLFilter() {
+    // Pattern type validation.
+    CHECK_NE(proto::URL_PATTERN_TYPE_UNSPECIFIED,
+             input_rule_.url_pattern_type());
+
+    // TODO(karandeepb): It would be nice to print the actual filter-list string
+    // in cases where rule conversion fails.
+    if (input_rule_.url_pattern_type() == proto::URL_PATTERN_TYPE_REGEXP) {
+      error_ = base::StringPrintf(
+          "Rule with filter %s ignored since regex rules are not supported.",
+          input_rule_.url_pattern().c_str());
+      return false;
+    }
+
+    std::string result;
+    switch (input_rule_.anchor_left()) {
+      case proto::ANCHOR_TYPE_NONE:
+        break;
+      case proto::ANCHOR_TYPE_BOUNDARY:
+        result += '|';
+        break;
+      case proto::ANCHOR_TYPE_SUBDOMAIN:
+        result += "||";
+        break;
+      case proto::ANCHOR_TYPE_UNSPECIFIED:
+        CHECK(false);
+        break;
+    }
+
+    result += input_rule_.url_pattern();
+
+    switch (input_rule_.anchor_right()) {
+      case proto::ANCHOR_TYPE_NONE:
+        break;
+      case proto::ANCHOR_TYPE_BOUNDARY:
+        result += '|';
+        break;
+      case proto::ANCHOR_TYPE_SUBDOMAIN:
+      case proto::ANCHOR_TYPE_UNSPECIFIED:
+        CHECK(false);
+        break;
+    }
+
+    // If |result| is empty, omit persisting the url pattern. In that case, it
+    // will match all urls.
+    if (!result.empty()) {
+      CHECK(json_rule_.SetPath({kRuleConditionKey, kUrlFilterKey},
+                               base::Value(result)));
+    }
+
+    return true;
+  }
+
+  bool PopulateIsURLFilterCaseSensitive() {
+    // Omit if case sensitive, since it's the default.
+    const bool case_sensitive = input_rule_.match_case();
+    if (case_sensitive)
+      return true;
+
+    CHECK(json_rule_.SetPath({kRuleConditionKey, kIsUrlFilterCaseSensitiveKey},
+                             base::Value(false)));
+    return true;
+  }
+
+  bool PopulateDomains() {
+    return PopulateDomainsInternal(kDomainsKey, false /*exclude_value*/);
+  }
+
+  bool PopulateExcludedDomains() {
+    return PopulateDomainsInternal(kExcludedDomainsKey, true /*exclude_value*/);
+  }
+
+  bool PopulateDomainsInternal(base::StringPiece sub_key, bool exclude_value) {
+    base::Value domains(base::Value::Type::LIST);
+
+    for (const proto::DomainListItem& item : input_rule_.domains()) {
+      if (item.exclude() == exclude_value)
+        domains.GetList().emplace_back(item.domain());
+    }
+
+    // Omit empty domain list.
+    if (!domains.GetList().empty()) {
+      CHECK(
+          json_rule_.SetPath({kRuleConditionKey, sub_key}, std::move(domains)));
+    }
+
+    return true;
+  }
+
+  bool PopulateResourceTypes() {
+    // Ensure that |element_types()| is a subset of proto::ElementType_ALL.
+    CHECK_EQ(proto::ELEMENT_TYPE_ALL,
+             proto::ELEMENT_TYPE_ALL | input_rule_.element_types());
+
+    base::Value resource_types(base::Value::Type::LIST);
+
+    int kMaskUnsupported =
+        proto::ELEMENT_TYPE_POPUP | proto::ELEMENT_TYPE_OBJECT_SUBREQUEST;
+
+    int element_mask = input_rule_.element_types() & (~kMaskUnsupported);
+
+    // We don't support object-subrequest. Instead let these be treated as rules
+    // matching object requests.
+    if (input_rule_.element_types() & proto::ELEMENT_TYPE_OBJECT_SUBREQUEST)
+      element_mask |= proto::ELEMENT_TYPE_OBJECT;
+
+    // No supported element types.
+    if (!element_mask) {
+      std::vector<std::string> element_types_removed;
+      if (input_rule_.element_types() & proto::ELEMENT_TYPE_POPUP)
+        element_types_removed.emplace_back("popup");
+      error_ = base::StringPrintf(
+          "Rule with filter %s and resource types [%s] ignored, no applicable "
+          "resource types",
+          input_rule_.url_pattern().c_str(),
+          base::JoinString(element_types_removed, "," /*separator*/).c_str());
+      return false;
+    }
+
+    // Omit resource types to block all subresources by default.
+    if (element_mask == (proto::ELEMENT_TYPE_ALL & ~kMaskUnsupported))
+      return true;
+
+    for (int element_type = 1; element_type <= proto::ElementType_MAX;
+         element_type <<= 1) {
+      CHECK(proto::ElementType_IsValid(element_type));
+
+      if (!(element_type & element_mask))
+        continue;
+
+      dnr_api::ResourceType resource_type = dnr_api::RESOURCE_TYPE_NONE;
+      switch (static_cast<proto::ElementType>(element_type)) {
+        case proto::ELEMENT_TYPE_UNSPECIFIED:
+          CHECK(false);
+          break;
+        case proto::ELEMENT_TYPE_OTHER:
+          resource_type = dnr_api::RESOURCE_TYPE_OTHER;
+          break;
+        case proto::ELEMENT_TYPE_SCRIPT:
+          resource_type = dnr_api::RESOURCE_TYPE_SCRIPT;
+          break;
+        case proto::ELEMENT_TYPE_IMAGE:
+          resource_type = dnr_api::RESOURCE_TYPE_IMAGE;
+          break;
+        case proto::ELEMENT_TYPE_STYLESHEET:
+          resource_type = dnr_api::RESOURCE_TYPE_STYLESHEET;
+          break;
+        case proto::ELEMENT_TYPE_OBJECT:
+          resource_type = dnr_api::RESOURCE_TYPE_OBJECT;
+          break;
+        case proto::ELEMENT_TYPE_XMLHTTPREQUEST:
+          resource_type = dnr_api::RESOURCE_TYPE_XMLHTTPREQUEST;
+          break;
+        case proto::ELEMENT_TYPE_OBJECT_SUBREQUEST:
+          // This was removed above.
+          CHECK(false);
+          break;
+        case proto::ELEMENT_TYPE_SUBDOCUMENT:
+          resource_type = dnr_api::RESOURCE_TYPE_SUB_FRAME;
+          break;
+        case proto::ELEMENT_TYPE_PING:
+          resource_type = dnr_api::RESOURCE_TYPE_PING;
+          break;
+        case proto::ELEMENT_TYPE_MEDIA:
+          resource_type = dnr_api::RESOURCE_TYPE_MEDIA;
+          break;
+        case proto::ELEMENT_TYPE_FONT:
+          resource_type = dnr_api::RESOURCE_TYPE_FONT;
+          break;
+        case proto::ELEMENT_TYPE_POPUP:
+          CHECK(false);
+          break;
+        case proto::ELEMENT_TYPE_WEBSOCKET:
+          resource_type = dnr_api::RESOURCE_TYPE_WEBSOCKET;
+          break;
+        case proto::ELEMENT_TYPE_ALL:
+          CHECK(false);
+          break;
+      }
+
+      resource_types.GetList().emplace_back(dnr_api::ToString(resource_type));
+    }
+
+    CHECK(json_rule_.SetPath({kRuleConditionKey, kResourceTypesKey},
+                             std::move(resource_types)));
+    return true;
+  }
+
+  bool PopulateExcludedResourceTypes() {
+    // We don't populate the "excludedResourceTypes" since that information has
+    // been processed away by conversion to a proto::UrlRule.
+    return true;
+  }
+
+  bool PopulateDomainType() {
+    dnr_api::DomainType domain_type = dnr_api::DOMAIN_TYPE_NONE;
+
+    switch (input_rule_.source_type()) {
+      case proto::SOURCE_TYPE_ANY:
+        // This is the default domain type and can be omitted.
+        return true;
+      case proto::SOURCE_TYPE_FIRST_PARTY:
+        domain_type = dnr_api::DOMAIN_TYPE_FIRSTPARTY;
+        break;
+      case proto::SOURCE_TYPE_THIRD_PARTY:
+        domain_type = dnr_api::DOMAIN_TYPE_THIRDPARTY;
+        break;
+      case proto::SOURCE_TYPE_UNSPECIFIED:
+        CHECK(false);
+        break;
+    }
+
+    CHECK_NE(dnr_api::DOMAIN_TYPE_NONE, domain_type);
+    CHECK(json_rule_.SetPath({kRuleConditionKey, kDomainTypeKey},
+                             base::Value(dnr_api::ToString(domain_type))));
+    return true;
+  }
+
+  bool PopulateRuleActionType() {
+    dnr_api::RuleActionType action_type = dnr_api::RULE_ACTION_TYPE_NONE;
+
+    switch (input_rule_.semantics()) {
+      case proto::RULE_SEMANTICS_BLACKLIST:
+        action_type = dnr_api::RULE_ACTION_TYPE_BLOCK;
+        break;
+      case proto::RULE_SEMANTICS_WHITELIST:
+        action_type = dnr_api::RULE_ACTION_TYPE_ALLOW;
+        break;
+      case proto::RULE_SEMANTICS_UNSPECIFIED:
+        CHECK(false);
+        break;
+    }
+
+    CHECK_NE(dnr_api::RULE_ACTION_TYPE_NONE, action_type);
+    CHECK(json_rule_.SetPath({kRuleActionKey, kRuleActionTypeKey},
+                             base::Value(dnr_api::ToString(action_type))));
+    return true;
+  }
+
+  bool PopulateRedirectURL() {
+    // Do nothing. The tool only supports allow and block rules.
+    return true;
+  }
+
+  bool PopulateRemoveHeadersList() {
+    // Do nothing. The tool only supports allow and block rules.
+    return true;
+  }
+
+  proto::UrlRule input_rule_;
+  int rule_id_;
+  std::string error_;
+  base::Value json_rule_;
+
+  DISALLOW_COPY_AND_ASSIGN(ProtoToJSONRuleConverter);
+};
+
+// Writes rules/extension to |output_path| in the format supported by
+// Declarative Net Request.
+class DNRJsonRuleOutputStream : public subresource_filter::RuleOutputStream {
+ public:
+  DNRJsonRuleOutputStream(const base::FilePath& output_path,
+                          filter_list_converter::WriteType type,
+                          bool noisy)
+      : rule_id_(kMinValidID),
+        output_rules_list_(base::Value::Type::LIST),
+        output_path_(output_path),
+        write_type_(type),
+        noisy_(noisy) {}
+
+  bool PutUrlRule(const proto::UrlRule& rule) override {
+    std::string error;
+    base::Value json_rule_value =
+        ProtoToJSONRuleConverter::Convert(rule, rule_id_, &error);
+
+    if (json_rule_value.is_none()) {
+      if (noisy_) {
+        LOG(ERROR) << base::StringPrintf("Error for id %d: %s", rule_id_,
+                                         error.c_str());
+      }
+      return false;
+    }
+
+    CHECK(error.empty());
+    CHECK(json_rule_value.is_dict());
+    output_rules_list_.GetList().push_back(std::move(json_rule_value));
+    ++rule_id_;
+    return true;
+  }
+
+  bool PutCssRule(const proto::CssRule& rule) override {
+    // Ignore CSS rules.
+    return true;
+  }
+
+  bool Finish() override {
+    switch (write_type_) {
+      case filter_list_converter::kExtension:
+        WriteManifestAndRuleset(output_path_, kJSONRulesetFilepath,
+                                kJSONRulesFilename, output_rules_list_,
+                                {} /* hosts */);
+        break;
+      case filter_list_converter::kJSONRuleset:
+        JSONFileValueSerializer(output_path_).Serialize(output_rules_list_);
+        break;
+    }
+
+    return true;
+  }
+
+ private:
+  int rule_id_ = kMinValidID;
+  base::Value output_rules_list_;
+  const base::FilePath output_path_;
+  const filter_list_converter::WriteType write_type_;
+  const bool noisy_;
+
+  DISALLOW_COPY_AND_ASSIGN(DNRJsonRuleOutputStream);
+};
+
+}  // namespace
+
+namespace filter_list_converter {
+
+bool ConvertRuleset(const std::vector<base::FilePath>& filter_list_inputs,
+                    const base::FilePath& output_path,
+                    WriteType type,
+                    bool noisy) {
+  DNRJsonRuleOutputStream rule_output_stream(output_path, type, noisy);
+
+  for (const auto& input_path : filter_list_inputs) {
+    auto rule_input_stream = subresource_filter::RuleInputStream::Create(
+        std::make_unique<std::ifstream>(input_path.AsUTF8Unsafe(),
+                                        std::ios::binary | std::ios::in),
+        subresource_filter::RulesetFormat::kFilterList);
+    CHECK(rule_input_stream);
+    CHECK(subresource_filter::TransferRules(rule_input_stream.get(),
+                                            &rule_output_stream,
+                                            nullptr /* css_rule_output */));
+  }
+
+  return rule_output_stream.Finish();
+}
+
+}  // namespace filter_list_converter
+}  // namespace declarative_net_request
+}  // namespace extensions

extensions/browser/api/declarative_net_request/filter_list_converter/converter.h

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef EXTENSIONS_BROWSER_API_DECLARATIVE_NET_REQUEST_FILTER_LIST_CONVERTER_CONVERTER_H_
+#define EXTENSIONS_BROWSER_API_DECLARATIVE_NET_REQUEST_FILTER_LIST_CONVERTER_CONVERTER_H_
+
+#include <vector>
+
+#include "base/files/file_path.h"
+#include "base/macros.h"
+
+namespace extensions {
+namespace declarative_net_request {
+namespace filter_list_converter {
+
+enum WriteType {
+  kJSONRuleset,
+  kExtension,
+};
+
+// Utility function to convert filter list files in the text format to a JSON
+// file in a format supported by the Declarative Net Request API. If |type| is
+// kExtension, output_path is treated as the extension directory and the ruleset
+// is written to "rules.json". Else it is treated as the json ruleset location.
+// Returns false if the conversion fails.
+bool ConvertRuleset(const std::vector<base::FilePath>& filter_list_inputs,
+                    const base::FilePath& output_path,
+                    WriteType type,
+                    bool noisy = true);
+
+}  // namespace filter_list_converter
+}  // namespace declarative_net_request
+}  // namespace extensions
+
+#endif  // EXTENSIONS_BROWSER_API_DECLARATIVE_NET_REQUEST_FILTER_LIST_CONVERTER_CONVERTER_H_

extensions/browser/api/declarative_net_request/filter_list_converter/converter_unittest.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "extensions/browser/api/declarative_net_request/filter_list_converter/converter.h"
+
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/files/scoped_temp_dir.h"
+#include "base/json/json_file_value_serializer.h"
+#include "base/json/json_reader.h"
+#include "base/logging.h"
+#include "base/optional.h"
+#include "base/strings/string_util.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace extensions {
+namespace declarative_net_request {
+namespace filter_list_converter {
+namespace {
+
+void TestConversion(std::vector<std::string> filter_list_rules,
+                    std::string json_result,
+                    WriteType write_type) {
+  base::ScopedTempDir temp_dir;
+  CHECK(temp_dir.CreateUniqueTempDir());
+
+  base::FilePath input_path = temp_dir.GetPath().AppendASCII("filterlist.txt");
+  std::string filterlist = base::JoinString(filter_list_rules, "\n");
+  CHECK_EQ(filterlist.size(),
+           static_cast<size_t>(base::WriteFile(input_path, filterlist.c_str(),
+                                               filterlist.size())));
+
+  base::Optional<base::Value> expected_json =
+      base::JSONReader::Read(json_result);
+  CHECK(expected_json.has_value());
+
+  base::FilePath output_path = temp_dir.GetPath();
+  if (write_type == WriteType::kJSONRuleset)
+    output_path = output_path.AppendASCII("rules.json");
+
+  ConvertRuleset({input_path}, output_path, write_type, false /* noisy */);
+
+  base::FilePath output_json_path =
+      temp_dir.GetPath().AppendASCII("rules.json");
+  JSONFileValueDeserializer deserializer(output_json_path);
+  std::unique_ptr<base::Value> actual_json = deserializer.Deserialize(
+      nullptr /* error_code */, nullptr /* error_message */);
+  ASSERT_TRUE(actual_json.get());
+
+  EXPECT_EQ(*expected_json, *actual_json);
+
+  if (write_type == WriteType::kExtension) {
+    EXPECT_TRUE(
+        base::PathExists(temp_dir.GetPath().AppendASCII("manifest.json")));
+  }
+}
+
+class FilterListConverterTest : public ::testing::TestWithParam<WriteType> {};
+
+TEST_P(FilterListConverterTest, Convert) {
+  std::vector<std::string> filter_list_rules = {
+      "||example.com^|$script,image,font",
+      "@@allowed.com$domain=example.com|~sub.example.com",
+      "|https://*.abc.com|$match-case,~image,third-party",
+      "abc.com$~third-party"};
+
+  std::string expected_result = R"(
+    [ {
+       "action": {
+          "type": "block"
+       },
+       "condition": {
+          "isUrlFilterCaseSensitive": false,
+          "resourceTypes": [ "script", "image", "font" ],
+          "urlFilter": "||example.com^|"
+       },
+       "id": 1
+    }, {
+       "action": {
+          "type": "allow"
+       },
+       "condition": {
+          "domains": [ "example.com" ],
+          "excludedDomains": [ "sub.example.com" ],
+          "isUrlFilterCaseSensitive": false,
+          "urlFilter": "allowed.com"
+       },
+       "id": 2
+    }, {
+       "action": {
+          "type": "block"
+       },
+       "condition": {
+          "resourceTypes": [ "other", "script", "stylesheet", "object",
+              "xmlhttprequest", "sub_frame", "ping", "media", "font",
+              "websocket" ],
+          "urlFilter": "|https://*.abc.com|",
+          "domainType": "thirdParty"
+       },
+       "id": 3
+    }, {
+       "action": {
+          "type": "block"
+       },
+       "condition": {
+          "isUrlFilterCaseSensitive": false,
+          "urlFilter": "abc.com",
+          "domainType": "firstParty"
+       },
+       "id": 4
+    } ]
+
+)";
+
+  TestConversion(filter_list_rules, expected_result, GetParam());
+}
+
+INSTANTIATE_TEST_SUITE_P(,
+                         FilterListConverterTest,
+                         ::testing::Values(WriteType::kExtension,
+                                           WriteType::kJSONRuleset));
+
+}  // namespace
+}  // namespace filter_list_converter
+}  // namespace declarative_net_request
+}  // namespace extensions

extensions/browser/api/declarative_net_request/filter_list_converter/main.cc

+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <vector>
+
+#include "base/command_line.h"
+#include "base/files/file_path.h"
+#include "base/files/file_util.h"
+#include "base/logging.h"
+#include "base/strings/string_split.h"
+#include "base/strings/stringprintf.h"
+#include "extensions/browser/api/declarative_net_request/filter_list_converter/converter.h"
+
+namespace {
+
+const char kSwitchInputFilterlistFiles[] = "input_filterlists";
+const char kSwitchOutputPath[] = "output_path";
+const char kSwitchOutputType[] = "output_type";
+const char kOutputTypeExtension[] = "extension";
+const char kOutputTypeJSON[] = "json";
+const char kJSONExtension[] = ".json";
+
+const char kHelpMsg[] = R"(
+  filter_list_converter --input_filterlists=[<path1>, <path2>]
+          --output_path=<path> --output_type=<extension,json>
+
+  Filter List Converter is a tool to convert filter list files in the text
+  format to a JSON file in a format supported by the Declarative Net Request
+  API. It can either output the complete extension or just the JSON ruleset.
+
+  --input_filterlists = List of input paths to text filter list files.
+  --output_path = The output path. The parent directory should exist.
+  --output_type = Optional switch. One of "extension" or "json". "json" is the
+                  default.
+)";
+
+namespace filter_list_converter =
+    extensions::declarative_net_request::filter_list_converter;
+
+void PrintHelp() {
+  LOG(ERROR) << kHelpMsg;
+}
+
+}  // namespace
+
+int main(int argc, char* argv[]) {
+  base::CommandLine::Init(argc, argv);
+  base::CommandLine& command_line = *base::CommandLine::ForCurrentProcess();
+
+  if (!command_line.HasSwitch(kSwitchInputFilterlistFiles) ||
+      !command_line.HasSwitch(kSwitchOutputPath)) {
+    PrintHelp();
+    return 1;
+  }
+
+  std::vector<base::FilePath> input_paths;
+  base::CommandLine::StringType comma_separated_paths =
+      command_line.GetSwitchValueNative(kSwitchInputFilterlistFiles);
+  for (const auto& piece : base::SplitStringPiece(
+           comma_separated_paths, "," /* separator */, base::TRIM_WHITESPACE,
+           base::SPLIT_WANT_NONEMPTY)) {
+    base::FilePath path(piece);
+
+    if (!base::PathExists(path)) {
+      LOG(ERROR) << "Input path " << piece << " does not exist.";
+      return 1;
+    }
+
+    input_paths.push_back(path);
+  }
+  if (input_paths.empty()) {
+    LOG(ERROR) << base::StringPrintf(
+        "No valid input files specified using '%s'.",
+        kSwitchInputFilterlistFiles);
+    return 1;
+  }
+
+  filter_list_converter::WriteType write_type =
+      filter_list_converter::kJSONRuleset;
+  if (command_line.HasSwitch(kSwitchOutputType)) {
+    std::string output_type =
+        command_line.GetSwitchValueASCII(kSwitchOutputType);
+    if (output_type == kOutputTypeExtension) {
+      write_type = filter_list_converter::kExtension;
+    } else if (output_type == kOutputTypeJSON) {
+      write_type = filter_list_converter::kJSONRuleset;
+    } else {
+      LOG(ERROR) << base::StringPrintf("Invalid value for switch '%s'",
+                                       kSwitchOutputType);
+      return 1;
+    }
+  }
+
+  base::FilePath output_path =
+      command_line.GetSwitchValuePath(kSwitchOutputPath);
+  bool invalid_output_path = false;
+  switch (write_type) {
+    case filter_list_converter::kExtension:
+      invalid_output_path = !base::DirectoryExists(output_path);
+      break;
+    case filter_list_converter::kJSONRuleset:
+      invalid_output_path = output_path.Extension() != kJSONExtension;
+      invalid_output_path |= !base::DirectoryExists(output_path.DirName());
+      break;
+  }
+  if (invalid_output_path) {
+    LOG(ERROR) << "Invalid output path " << output_path.value();
+    return 1;
+  }
+
+  if (!filter_list_converter::ConvertRuleset(input_paths, output_path,
+                                             write_type)) {
+    LOG(ERROR) << "Conversion failed.";
+    return 1;
+  }
+
+  return 0;
+}

extensions/browser/api/declarative_net_request/indexed_rule.cc

@@ -272,7 +272,6 @@ ParseResult IndexedRule::CreateIndexedRule(dnr_api::Rule parsed_rule,
                                            const GURL& base_url,
                                            IndexedRule* indexed_rule) {
   DCHECK(indexed_rule);
-  DCHECK(IsAPIAvailable());
 
   if (parsed_rule.id < kMinValidID)
     return ParseResult::ERROR_INVALID_RULE_ID;

extensions/browser/api/declarative_net_request/ruleset_source.cc

@@ -367,6 +367,7 @@ ParseInfo RulesetSource::IndexAndPersistRules(std::vector<dnr_api::Rule> rules,
                                               int* ruleset_checksum) const {
   DCHECK_LE(rules.size(), rule_count_limit_);
   DCHECK(ruleset_checksum);
+  DCHECK(IsAPIAvailable());
 
   FlatRulesetIndexer indexer;