Support for JSON input conversion

This adds support for reading JSON files in GN and converting them
to equivalent GN values.

Change-Id: I5bca3abb4668da852774d69c98d4dd6575a42957
Reviewed-on: https://chromium-review.googlesource.com/894857
Commit-Queue: Petr Hosek <phosek@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Reviewed-by: Brett Wilson <brettw@chromium.org>
Cr-Commit-Position: refs/heads/master@{#535078}

tools/gn/docs/reference.md

@@ -5872,6 +5872,20 @@
       Note that if the input is empty, the result will be a null value which
       will produce an error if assigned to a variable.
 
+  "json"
+      Parse the input as a JSON and convert it to equivalent GN rvalue. The data
+      type mapping is:
+        a string in JSON maps to string in GN
+        an integer in JSON maps to integer in GN
+        a float in JSON is unsupported and will result in an error
+        an object in JSON maps to scope in GN
+        an array in JSON maps to list in GN
+        a boolean in JSON maps to boolean in GN
+        a null in JSON is unsupported and will result in an error
+
+      Nota that the dictionary keys have to be valid GN identifiers otherwise
+      they will produce an error.
+
   "trim ..."
       Prefixing any of the other transformations with the word "trim" will
       result in whitespace being trimmed from the beginning and end of the

tools/gn/input_conversion.cc

@@ -7,9 +7,11 @@
 #include <memory>
 #include <utility>
 
+#include "base/json/json_reader.h"
 #include "base/macros.h"
 #include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
+#include "base/values.h"
 #include "tools/gn/build_settings.h"
 #include "tools/gn/err.h"
 #include "tools/gn/input_file.h"
@@ -106,6 +108,98 @@ Value ParseList(const std::string& input, const ParseNode* origin, Err* err) {
   return ret;
 }
 
+bool IsIdentifier(const base::StringPiece& buffer) {
+  DCHECK(buffer.size() > 0);
+  if (!Tokenizer::IsIdentifierFirstChar(buffer[0]))
+    return false;
+  for (size_t i = 1; i < buffer.size(); i++)
+    if (!Tokenizer::IsIdentifierContinuingChar(buffer[i]))
+      return false;
+  return true;
+}
+
+Value ParseJSONValue(const Settings* settings,
+                     const base::Value& value,
+                     const ParseNode* origin,
+                     InputFile* input_file,
+                     Err* err) {
+  switch (value.type()) {
+    case base::Value::Type::NONE:
+      *err = Err(origin, "Null values are not supported.");
+      return Value();
+    case base::Value::Type::BOOLEAN:
+      return Value(origin, value.GetBool());
+    case base::Value::Type::INTEGER:
+      return Value(origin, static_cast<int64_t>(value.GetInt()));
+    case base::Value::Type::DOUBLE:
+      *err = Err(origin, "Floating point values are not supported.");
+      return Value();
+    case base::Value::Type::STRING:
+      return Value(origin, value.GetString());
+    case base::Value::Type::BINARY:
+      *err = Err(origin, "Binary values are not supported.");
+      return Value();
+    case base::Value::Type::DICTIONARY: {
+      std::unique_ptr<Scope> scope = std::make_unique<Scope>(settings);
+      for (const auto& it : value.DictItems()) {
+        Value value =
+            ParseJSONValue(settings, it.second, origin, input_file, err);
+        if (!IsIdentifier(it.first)) {
+          *err = Err(origin, "Invalid identifier \"" + it.first + "\".");
+          return Value();
+        }
+        // Search for the key in the input file. We know it's present because
+        // it was parsed by the JSON reader, but we need its location to
+        // construct a StringPiece that can be used as key in the Scope.
+        size_t off = input_file->contents().find("\"" + it.first + "\"");
+        if (off == std::string::npos) {
+          *err = Err(origin, "Invalid encoding \"" + it.first + "\".");
+          return Value();
+        }
+        base::StringPiece key(&input_file->contents()[off + 1],
+                              it.first.size());
+        scope->SetValue(key, std::move(value), origin);
+      }
+      return Value(origin, std::move(scope));
+    }
+    case base::Value::Type::LIST: {
+      Value result(origin, Value::LIST);
+      result.list_value().reserve(value.GetList().size());
+      for (const auto& val : value.GetList()) {
+        Value value = ParseJSONValue(settings, val, origin, input_file, err);
+        result.list_value().push_back(value);
+      }
+      return result;
+    }
+  }
+  return Value();
+}
+
+// Parses the JSON string and converts it to GN value.
+Value ParseJSON(const Settings* settings,
+                const std::string& input,
+                const ParseNode* origin,
+                Err* err) {
+  InputFile* input_file;
+  std::vector<Token>* tokens;
+  std::unique_ptr<ParseNode>* parse_root_ptr;
+  g_scheduler->input_file_manager()->AddDynamicInput(SourceFile(), &input_file,
+                                                     &tokens, &parse_root_ptr);
+  input_file->SetContents(input);
+
+  int error_code_out;
+  std::string error_msg_out;
+  std::unique_ptr<base::Value> value = base::JSONReader::ReadAndReturnError(
+      input, base::JSONParserOptions::JSON_PARSE_RFC, &error_code_out,
+      &error_msg_out);
+  if (!value) {
+    *err = Err(origin, "Input is not a valid JSON: " + error_msg_out);
+    return Value();
+  }
+
+  return ParseJSONValue(settings, *value, origin, input_file, err);
+}
+
 // Backend for ConvertInputToValue, this takes the extracted string for the
 // input conversion so we can recursively call ourselves to handle the optional
 // "trim" prefix. This original value is also kept for the purposes of throwing
@@ -139,6 +233,8 @@ Value DoConvertInputToValue(const Settings* settings,
     return ParseList(input, origin, err);
   if (input_conversion == "scope")
     return ParseValueOrScope(settings, input, PARSE_SCOPE, origin, err);
+  if (input_conversion == "json")
+    return ParseJSON(settings, input, origin, err);
 
   *err = Err(original_input_conversion, "Not a valid input_conversion.",
              "Run gn help input_conversion to see your options.");
@@ -189,6 +285,20 @@ const char kInputConversion_Help[] =
       Note that if the input is empty, the result will be a null value which
       will produce an error if assigned to a variable.
 
+  "json"
+      Parse the input as a JSON and convert it to equivalent GN rvalue. The data
+      type mapping is:
+        a string in JSON maps to string in GN
+        an integer in JSON maps to integer in GN
+        a float in JSON is unsupported and will result in an error
+        an object in JSON maps to scope in GN
+        an array in JSON maps to list in GN
+        a boolean in JSON maps to boolean in GN
+        a null in JSON is unsupported and will result in an error
+
+      Nota that the dictionary keys have to be valid GN identifiers otherwise
+      they will produce an error.
+
   "trim ..."
       Prefixing any of the other transformations with the word "trim" will
       result in whitespace being trimmed from the beginning and end of the

tools/gn/input_conversion_unittest.cc

@@ -132,6 +132,98 @@ TEST_F(InputConversionTest, ValueDict) {
   EXPECT_EQ(input, a_file->contents());
 }
 
+TEST_F(InputConversionTest, ValueJSON) {
+  Err err;
+  std::string input(R"*({
+  "a": 5,
+  "b": "foo",
+  "c": {
+    "d": true,
+    "e": [
+      {
+        "f": "bar"
+      }
+    ]
+  }
+})*");
+  Value result = ConvertInputToValue(settings(), input, nullptr,
+                                     Value(nullptr, "json"), &err);
+  EXPECT_FALSE(err.has_error());
+  ASSERT_EQ(Value::SCOPE, result.type());
+
+  const Value* a_value = result.scope_value()->GetValue("a");
+  ASSERT_TRUE(a_value);
+  EXPECT_EQ(5, a_value->int_value());
+
+  const Value* b_value = result.scope_value()->GetValue("b");
+  ASSERT_TRUE(b_value);
+  EXPECT_EQ("foo", b_value->string_value());
+
+  const Value* c_value = result.scope_value()->GetValue("c");
+  ASSERT_TRUE(c_value);
+  ASSERT_EQ(Value::SCOPE, c_value->type());
+
+  const Value* d_value = c_value->scope_value()->GetValue("d");
+  ASSERT_TRUE(d_value);
+  EXPECT_EQ(true, d_value->boolean_value());
+
+  const Value* e_value = c_value->scope_value()->GetValue("e");
+  ASSERT_TRUE(e_value);
+  ASSERT_EQ(Value::LIST, e_value->type());
+
+  EXPECT_EQ(1u, e_value->list_value().size());
+  ASSERT_EQ(Value::SCOPE, e_value->list_value()[0].type());
+  const Value* f_value = e_value->list_value()[0].scope_value()->GetValue("f");
+  ASSERT_TRUE(f_value);
+  EXPECT_EQ("bar", f_value->string_value());
+}
+
+TEST_F(InputConversionTest, ValueJSONInvalidInput) {
+  Err err;
+  std::string input(R"*({
+  "a": 5,
+  "b":
+})*");
+  Value result = ConvertInputToValue(settings(), input, nullptr,
+                                     Value(nullptr, "json"), &err);
+  EXPECT_TRUE(err.has_error());
+  EXPECT_EQ("Input is not a valid JSON: Line: 4, column: 2, Unexpected token.",
+            err.message());
+}
+
+TEST_F(InputConversionTest, ValueJSONUnsupportedValue) {
+  Err err;
+  std::string input(R"*({
+  "a": null
+})*");
+  Value result = ConvertInputToValue(settings(), input, nullptr,
+                                     Value(nullptr, "json"), &err);
+  EXPECT_TRUE(err.has_error());
+  EXPECT_EQ("Null values are not supported.", err.message());
+}
+
+TEST_F(InputConversionTest, ValueJSONInvalidVariable) {
+  Err err;
+  std::string input(R"*({
+  "a\\x0001b": 5
+})*");
+  Value result = ConvertInputToValue(settings(), input, nullptr,
+                                     Value(nullptr, "json"), &err);
+  EXPECT_TRUE(err.has_error());
+  EXPECT_EQ("Invalid identifier \"a\\x0001b\".", err.message());
+}
+
+TEST_F(InputConversionTest, ValueJSONUnsupported) {
+  Err err;
+  std::string input(R"*({
+  "d": 0.0
+})*");
+  Value result = ConvertInputToValue(settings(), input, nullptr,
+                                     Value(nullptr, "json"), &err);
+  EXPECT_TRUE(err.has_error());
+  EXPECT_EQ("Floating point values are not supported.", err.message());
+}
+
 TEST_F(InputConversionTest, ValueEmpty) {
   Err err;
   Value result = ConvertInputToValue(settings(), "", nullptr,