[gn] Ensure bots correctly parse $root_build_dir/args.gn

The downstream iOS bots import a gn file from $root_build_dir/args.gn
that would not be correctly parsed by //build/gn_helpers.py as:

1.  define some values of type scope (`{ a = "value"}`),
2.  it uses comments in many places (before all variables that are
    defined, above individual items in lists, ...).

To ensure that the file $root_build_dir/args.gn could be parsed, fix
//build/gn_helpers.py to support the scope values and to consider
comments as valid in any location where whitespace would have been.

Bug: 1144711
Change-Id: I0c48167f1bc818dc9f6253287c2c967d13756523
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2513217
Commit-Queue: Sylvain Defresne <sdefresne@chromium.org>
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Auto-Submit: Sylvain Defresne <sdefresne@chromium.org>
Cr-Commit-Position: refs/heads/master@{#823230}

build/gn_helpers.py

@@ -289,13 +289,12 @@ class GNValueParser(object):
     self.ReplaceImports()
 
 
-  def ConsumeWhitespace(self):
+  def _ConsumeWhitespace(self):
     while not self.IsDone() and self.input[self.cur] in ' \t\n':
       self.cur += 1
 
-  def ConsumeComment(self):
-    if self.IsDone() or self.input[self.cur] != '#':
-      return
+  def ConsumeCommentAndWhitespace(self):
+    self._ConsumeWhitespace()
 
     # Consume each comment, line by line.
     while not self.IsDone() and self.input[self.cur] == '#':
@@ -306,6 +305,8 @@ class GNValueParser(object):
       if not self.IsDone():
         self.cur += 1
 
+      self._ConsumeWhitespace()
+
   def Parse(self):
     """Converts a string representing a printed GN value to the Python type.
 
@@ -328,7 +329,7 @@ class GNValueParser(object):
       GNError: Parse fails.
     """
     result = self._ParseAllowTrailing()
-    self.ConsumeWhitespace()
+    self.ConsumeCommentAndWhitespace()
     if not self.IsDone():
       raise GNError("Trailing input after parsing:\n  " + self.input[self.cur:])
     return result
@@ -344,32 +345,32 @@ class GNValueParser(object):
     d = {}
 
     self.ReplaceImports()
-    self.ConsumeWhitespace()
-    self.ConsumeComment()
+    self.ConsumeCommentAndWhitespace()
+
     while not self.IsDone():
       ident = self._ParseIdent()
-      self.ConsumeWhitespace()
+      self.ConsumeCommentAndWhitespace()
       if self.input[self.cur] != '=':
         raise GNError("Unexpected token: " + self.input[self.cur:])
       self.cur += 1
-      self.ConsumeWhitespace()
+      self.ConsumeCommentAndWhitespace()
       val = self._ParseAllowTrailing()
-      self.ConsumeWhitespace()
-      self.ConsumeComment()
-      self.ConsumeWhitespace()
+      self.ConsumeCommentAndWhitespace()
       d[ident] = val
 
     return d
 
   def _ParseAllowTrailing(self):
     """Internal version of Parse() that doesn't check for trailing stuff."""
-    self.ConsumeWhitespace()
+    self.ConsumeCommentAndWhitespace()
     if self.IsDone():
       raise GNError("Expected input to parse.")
 
     next_char = self.input[self.cur]
     if next_char == '[':
       return self.ParseList()
+    elif next_char == '{':
+      return self.ParseScope()
     elif _IsDigitOrMinus(next_char):
       return self.ParseNumber()
     elif next_char == '"':
@@ -400,7 +401,7 @@ class GNValueParser(object):
     return ident
 
   def ParseNumber(self):
-    self.ConsumeWhitespace()
+    self.ConsumeCommentAndWhitespace()
     if self.IsDone():
       raise GNError('Expected number but got nothing.')
 
@@ -418,7 +419,7 @@ class GNValueParser(object):
     return int(number_string)
 
   def ParseString(self):
-    self.ConsumeWhitespace()
+    self.ConsumeCommentAndWhitespace()
     if self.IsDone():
       raise GNError('Expected string but got nothing.')
 
@@ -444,7 +445,7 @@ class GNValueParser(object):
     return UnescapeGNString(self.input[begin:end])
 
   def ParseList(self):
-    self.ConsumeWhitespace()
+    self.ConsumeCommentAndWhitespace()
     if self.IsDone():
       raise GNError('Expected list but got nothing.')
 
@@ -452,7 +453,7 @@ class GNValueParser(object):
     if self.input[self.cur] != '[':
       raise GNError('Expected [ for list but got:\n  ' + self.input[self.cur:])
     self.cur += 1
-    self.ConsumeWhitespace()
+    self.ConsumeCommentAndWhitespace()
     if self.IsDone():
       raise GNError('Unterminated list:\n  ' + self.input)
 
@@ -467,7 +468,7 @@ class GNValueParser(object):
         raise GNError('List items not separated by comma.')
 
       list_result += [ self._ParseAllowTrailing() ]
-      self.ConsumeWhitespace()
+      self.ConsumeCommentAndWhitespace()
       if self.IsDone():
         break
 
@@ -476,10 +477,41 @@ class GNValueParser(object):
       if previous_had_trailing_comma:
         # Consume comma.
         self.cur += 1
-        self.ConsumeWhitespace()
+        self.ConsumeCommentAndWhitespace()
 
     raise GNError('Unterminated list:\n  ' + self.input)
 
+  def ParseScope(self):
+    self.ConsumeCommentAndWhitespace()
+    if self.IsDone():
+      raise GNError('Expected scope but got nothing.')
+
+    # Skip over opening '{'.
+    if self.input[self.cur] != '{':
+      raise GNError('Expected { for scope but got:\n ' + self.input[self.cur:])
+    self.cur += 1
+    self.ConsumeCommentAndWhitespace()
+    if self.IsDone():
+      raise GNError('Unterminated scope:\n ' + self.input)
+
+    scope_result = {}
+    while not self.IsDone():
+      if self.input[self.cur] == '}':
+        self.cur += 1
+        return scope_result
+
+      ident = self._ParseIdent()
+      self.ConsumeCommentAndWhitespace()
+      if self.input[self.cur] != '=':
+        raise GNError("Unexpected token: " + self.input[self.cur:])
+      self.cur += 1
+      self.ConsumeCommentAndWhitespace()
+      val = self._ParseAllowTrailing()
+      self.ConsumeCommentAndWhitespace()
+      scope_result[ident] = val
+
+    raise GNError('Unterminated scope:\n ' + self.input)
+
   def _ConstantFollows(self, constant):
     """Checks and maybe consumes a string constant at current input location.
 

build/gn_helpers_unittest.py

@@ -128,6 +128,26 @@ class UnitTest(unittest.TestCase):
       parser = gn_helpers.GNValueParser('[1 2]')  # No separating comma.
       parser.ParseList()
 
+  def test_ParseScope(self):
+    parser = gn_helpers.GNValueParser('{a = 1}')
+    self.assertEqual(parser.ParseScope(), {'a': 1})
+
+    with self.assertRaises(gn_helpers.GNError):
+      parser = gn_helpers.GNValueParser('')  # Empty.
+      parser.ParseScope()
+    with self.assertRaises(gn_helpers.GNError):
+      parser = gn_helpers.GNValueParser('asdf')  # No {}.
+      parser.ParseScope()
+    with self.assertRaises(gn_helpers.GNError):
+      parser = gn_helpers.GNValueParser('{a = 1')  # Unterminated.
+      parser.ParseScope()
+    with self.assertRaises(gn_helpers.GNError):
+      parser = gn_helpers.GNValueParser('{"a" = 1}')  # Not identifier.
+      parser.ParseScope()
+    with self.assertRaises(gn_helpers.GNError):
+      parser = gn_helpers.GNValueParser('{a = }')  # No value.
+      parser.ParseScope()
+
   def test_FromGNArgs(self):
     # Booleans and numbers should work; whitespace is allowed works.
     self.assertEqual(gn_helpers.FromGNArgs('foo = true\nbar = 1\n'),
@@ -159,6 +179,51 @@ class UnitTest(unittest.TestCase):
     self.assertEqual(gn_helpers.FromGNArgs(''), {})
     self.assertEqual(gn_helpers.FromGNArgs(' \n '), {})
 
+    # Comments should work everywhere (and be ignored).
+    gn_args_lines = [
+        '# Top-level comment.',
+        '',
+        '# Variable comment.',
+        'foo = true',
+        'bar = [',
+        '    # Value comment in list.',
+        '    1,',
+        '    2,',
+        ']',
+        '',
+        'baz # Comment anywhere, really',
+        '  = # also here',
+        '    4',
+    ]
+    self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)), {
+        'foo': True,
+        'bar': [1, 2],
+        'baz': 4
+    })
+
+    # Scope should be parsed, even empty ones.
+    gn_args_lines = [
+        'foo = {',
+        '  a = 1',
+        '  b = [',
+        '    { },',
+        '    {',
+        '      c = 1',
+        '    },',
+        '  ]',
+        '}',
+    ]
+    self.assertEqual(gn_helpers.FromGNArgs('\n'.join(gn_args_lines)),
+                     {'foo': {
+                         'a': 1,
+                         'b': [
+                             {},
+                             {
+                                 'c': 1,
+                             },
+                         ]
+                     }})
+
     # Non-identifiers should raise an exception.
     with self.assertRaises(gn_helpers.GNError):
       gn_helpers.FromGNArgs('123 = true')