gn: Reduce stack memory use while parsing.

BUG=641454

Review-Url: https://codereview.chromium.org/2290713002
Cr-Commit-Position: refs/heads/master@{#415181}

tools/gn/parser.cc

@@ -281,7 +281,9 @@ ParserHelper Parser::expressions_[] = {
 };
 
 Parser::Parser(const std::vector<Token>& tokens, Err* err)
-    : err_(err), cur_(0) {
+    : invalid_token_(Location(), Token::INVALID, base::StringPiece()),
+      err_(err),
+      cur_(0) {
   for (const auto& token : tokens) {
     switch (token.type()) {
       case Token::LINE_COMMENT:
@@ -377,20 +379,20 @@ bool Parser::Match(Token::Type type) {
   return true;
 }
 
-Token Parser::Consume(Token::Type type, const char* error_message) {
+const Token& Parser::Consume(Token::Type type, const char* error_message) {
   Token::Type types[1] = { type };
   return Consume(types, 1, error_message);
 }
 
-Token Parser::Consume(Token::Type* types,
-                      size_t num_types,
-                      const char* error_message) {
+const Token& Parser::Consume(Token::Type* types,
+                             size_t num_types,
+                             const char* error_message) {
   if (has_error()) {
     // Don't overwrite current error, but make progress through tokens so that
     // a loop that's expecting a particular token will still terminate.
     if (!at_end())
       cur_++;
-    return Token(Location(), Token::INVALID, base::StringPiece());
+    return invalid_token_;
   }
   if (at_end()) {
     const char kEOFMsg[] = "I hit EOF instead.";
@@ -398,7 +400,7 @@ Token Parser::Consume(Token::Type* types,
       *err_ = Err(Location(), error_message, kEOFMsg);
     else
       *err_ = Err(tokens_[tokens_.size() - 1], error_message, kEOFMsg);
-    return Token(Location(), Token::INVALID, base::StringPiece());
+    return invalid_token_;
   }
 
   for (size_t i = 0; i < num_types; ++i) {
@@ -406,10 +408,10 @@ Token Parser::Consume(Token::Type* types,
       return Consume();
   }
   *err_ = Err(cur_token(), error_message);
-  return Token(Location(), Token::INVALID, base::StringPiece());
+  return invalid_token_;
 }
 
-Token Parser::Consume() {
+const Token& Parser::Consume() {
   return tokens_[cur_++];
 }
 
@@ -421,7 +423,7 @@ std::unique_ptr<ParseNode> Parser::ParseExpression(int precedence) {
   if (at_end())
     return std::unique_ptr<ParseNode>();
 
-  Token token = Consume();
+  const Token& token = Consume();
   PrefixFunc prefix = expressions_[token.type()].prefix;
 
   if (prefix == nullptr) {
@@ -437,15 +439,15 @@ std::unique_ptr<ParseNode> Parser::ParseExpression(int precedence) {
 
   while (!at_end() && !IsStatementBreak(cur_token().type()) &&
          precedence <= expressions_[cur_token().type()].precedence) {
-    token = Consume();
-    InfixFunc infix = expressions_[token.type()].infix;
+    const Token& next_token = Consume();
+    InfixFunc infix = expressions_[next_token.type()].infix;
     if (infix == nullptr) {
-      *err_ = Err(token,
-                  std::string("Unexpected token '") +
-                      token.value().as_string() + std::string("'"));
+      *err_ = Err(next_token, std::string("Unexpected token '") +
+                                  next_token.value().as_string() +
+                                  std::string("'"));
       return std::unique_ptr<ParseNode>();
     }
-    left = (this->*infix)(std::move(left), token);
+    left = (this->*infix)(std::move(left), next_token);
     if (has_error())
       return std::unique_ptr<ParseNode>();
   }
@@ -453,27 +455,27 @@ std::unique_ptr<ParseNode> Parser::ParseExpression(int precedence) {
   return left;
 }
 
-std::unique_ptr<ParseNode> Parser::Block(Token token) {
+std::unique_ptr<ParseNode> Parser::Block(const Token& token) {
   // This entrypoint into ParseBlock means it's part of an expression and we
   // always want the result.
   return ParseBlock(token, BlockNode::RETURNS_SCOPE);
 }
 
-std::unique_ptr<ParseNode> Parser::Literal(Token token) {
+std::unique_ptr<ParseNode> Parser::Literal(const Token& token) {
   return base::WrapUnique(new LiteralNode(token));
 }
 
-std::unique_ptr<ParseNode> Parser::Name(Token token) {
+std::unique_ptr<ParseNode> Parser::Name(const Token& token) {
   return IdentifierOrCall(std::unique_ptr<ParseNode>(), token);
 }
 
-std::unique_ptr<ParseNode> Parser::BlockComment(Token token) {
+std::unique_ptr<ParseNode> Parser::BlockComment(const Token& token) {
   std::unique_ptr<BlockCommentNode> comment(new BlockCommentNode());
   comment->set_comment(token);
   return std::move(comment);
 }
 
-std::unique_ptr<ParseNode> Parser::Group(Token token) {
+std::unique_ptr<ParseNode> Parser::Group(const Token& token) {
   std::unique_ptr<ParseNode> expr = ParseExpression();
   if (has_error())
     return std::unique_ptr<ParseNode>();
@@ -481,7 +483,7 @@ std::unique_ptr<ParseNode> Parser::Group(Token token) {
   return expr;
 }
 
-std::unique_ptr<ParseNode> Parser::Not(Token token) {
+std::unique_ptr<ParseNode> Parser::Not(const Token& token) {
   std::unique_ptr<ParseNode> expr = ParseExpression(PRECEDENCE_PREFIX + 1);
   if (has_error())
     return std::unique_ptr<ParseNode>();
@@ -496,7 +498,7 @@ std::unique_ptr<ParseNode> Parser::Not(Token token) {
   return std::move(unary_op);
 }
 
-std::unique_ptr<ParseNode> Parser::List(Token node) {
+std::unique_ptr<ParseNode> Parser::List(const Token& node) {
   std::unique_ptr<ParseNode> list(ParseList(node, Token::RIGHT_BRACKET, true));
   if (!has_error() && !at_end())
     Consume(Token::RIGHT_BRACKET, "Expected ']'");
@@ -505,7 +507,7 @@ std::unique_ptr<ParseNode> Parser::List(Token node) {
 
 std::unique_ptr<ParseNode> Parser::BinaryOperator(
     std::unique_ptr<ParseNode> left,
-    Token token) {
+    const Token& token) {
   std::unique_ptr<ParseNode> right =
       ParseExpression(expressions_[token.type()].precedence + 1);
   if (!right) {
@@ -524,14 +526,14 @@ std::unique_ptr<ParseNode> Parser::BinaryOperator(
 
 std::unique_ptr<ParseNode> Parser::IdentifierOrCall(
     std::unique_ptr<ParseNode> left,
-    Token token) {
+    const Token& token) {
   std::unique_ptr<ListNode> list(new ListNode);
   list->set_begin_token(token);
   list->set_end(base::WrapUnique(new EndNode(token)));
   std::unique_ptr<BlockNode> block;
   bool has_arg = false;
   if (LookAhead(Token::LEFT_PAREN)) {
-    Token start_token = Consume();
+    const Token& start_token = Consume();
     // Parsing a function call.
     has_arg = true;
     if (Match(Token::RIGHT_PAREN)) {
@@ -563,7 +565,7 @@ std::unique_ptr<ParseNode> Parser::IdentifierOrCall(
 }
 
 std::unique_ptr<ParseNode> Parser::Assignment(std::unique_ptr<ParseNode> left,
-                                              Token token) {
+                                              const Token& token) {
   if (left->AsIdentifier() == nullptr && left->AsAccessor() == nullptr) {
     *err_ = Err(left.get(),
         "The left-hand side of an assignment must be an identifier, "
@@ -584,7 +586,7 @@ std::unique_ptr<ParseNode> Parser::Assignment(std::unique_ptr<ParseNode> left,
 }
 
 std::unique_ptr<ParseNode> Parser::Subscript(std::unique_ptr<ParseNode> left,
-                                             Token token) {
+                                             const Token& token) {
   // TODO: Maybe support more complex expressions like a[0][0]. This would
   // require work on the evaluator too.
   if (left->AsIdentifier() == nullptr) {
@@ -603,7 +605,7 @@ std::unique_ptr<ParseNode> Parser::Subscript(std::unique_ptr<ParseNode> left,
 }
 
 std::unique_ptr<ParseNode> Parser::DotOperator(std::unique_ptr<ParseNode> left,
-                                               Token token) {
+                                               const Token& token) {
   if (left->AsIdentifier() == nullptr) {
     *err_ = Err(left.get(), "May only use \".\" for identifiers.",
         "The thing on the left hand side of the dot must be an identifier\n"
@@ -627,7 +629,7 @@ std::unique_ptr<ParseNode> Parser::DotOperator(std::unique_ptr<ParseNode> left,
 }
 
 // Does not Consume the start or end token.
-std::unique_ptr<ListNode> Parser::ParseList(Token start_token,
+std::unique_ptr<ListNode> Parser::ParseList(const Token& start_token,
                                             Token::Type stop_before,
                                             bool allow_trailing_comma) {
   std::unique_ptr<ListNode> list(new ListNode);
@@ -709,7 +711,7 @@ std::unique_ptr<ParseNode> Parser::ParseStatement() {
         return stmt;
     }
     if (!has_error()) {
-      Token token = cur_or_last_token();
+      const Token& token = cur_or_last_token();
       *err_ = Err(token, "Expecting assignment or function call.");
     }
     return std::unique_ptr<ParseNode>();
@@ -717,7 +719,7 @@ std::unique_ptr<ParseNode> Parser::ParseStatement() {
 }
 
 std::unique_ptr<BlockNode> Parser::ParseBlock(
-    Token begin_brace,
+    const Token& begin_brace,
     BlockNode::ResultMode result_mode) {
   if (has_error())
     return std::unique_ptr<BlockNode>();

tools/gn/parser.h

@@ -17,9 +17,9 @@
 #include "tools/gn/parse_tree.h"
 
 class Parser;
-typedef std::unique_ptr<ParseNode> (Parser::*PrefixFunc)(Token token);
+typedef std::unique_ptr<ParseNode> (Parser::*PrefixFunc)(const Token& token);
 typedef std::unique_ptr<ParseNode> (
-    Parser::*InfixFunc)(std::unique_ptr<ParseNode> left, Token token);
+    Parser::*InfixFunc)(std::unique_ptr<ParseNode> left, const Token& token);
 
 extern const char kGrammar_Help[];
 
@@ -58,29 +58,29 @@ class Parser {
   std::unique_ptr<ParseNode> ParseExpression(int precedence);
 
   // |PrefixFunc|s used in parsing expressions.
-  std::unique_ptr<ParseNode> Block(Token token);
-  std::unique_ptr<ParseNode> Literal(Token token);
-  std::unique_ptr<ParseNode> Name(Token token);
-  std::unique_ptr<ParseNode> Group(Token token);
-  std::unique_ptr<ParseNode> Not(Token token);
-  std::unique_ptr<ParseNode> List(Token token);
-  std::unique_ptr<ParseNode> BlockComment(Token token);
+  std::unique_ptr<ParseNode> Block(const Token& token);
+  std::unique_ptr<ParseNode> Literal(const Token& token);
+  std::unique_ptr<ParseNode> Name(const Token& token);
+  std::unique_ptr<ParseNode> Group(const Token& token);
+  std::unique_ptr<ParseNode> Not(const Token& token);
+  std::unique_ptr<ParseNode> List(const Token& token);
+  std::unique_ptr<ParseNode> BlockComment(const Token& token);
 
   // |InfixFunc|s used in parsing expressions.
   std::unique_ptr<ParseNode> BinaryOperator(std::unique_ptr<ParseNode> left,
-                                            Token token);
+                                            const Token& token);
   std::unique_ptr<ParseNode> IdentifierOrCall(std::unique_ptr<ParseNode> left,
-                                              Token token);
+                                              const Token& token);
   std::unique_ptr<ParseNode> Assignment(std::unique_ptr<ParseNode> left,
-                                        Token token);
+                                        const Token& token);
   std::unique_ptr<ParseNode> Subscript(std::unique_ptr<ParseNode> left,
-                                       Token token);
+                                       const Token& token);
   std::unique_ptr<ParseNode> DotOperator(std::unique_ptr<ParseNode> left,
-                                         Token token);
+                                         const Token& token);
 
   // Helper to parse a comma separated list, optionally allowing trailing
   // commas (allowed in [] lists, not in function calls).
-  std::unique_ptr<ListNode> ParseList(Token start_token,
+  std::unique_ptr<ListNode> ParseList(const Token& start_token,
                                       Token::Type stop_before,
                                       bool allow_trailing_comma);
 
@@ -88,7 +88,7 @@ class Parser {
   std::unique_ptr<ParseNode> ParseStatement();
   // Expects to be passed the token corresponding to the '{' and that the
   // current token is the one following the '{'.
-  std::unique_ptr<BlockNode> ParseBlock(Token being_brace,
+  std::unique_ptr<BlockNode> ParseBlock(const Token& begin_brace,
                                         BlockNode::ResultMode result_mode);
   std::unique_ptr<ParseNode> ParseCondition();
 
@@ -105,11 +105,11 @@ class Parser {
 
   bool LookAhead(Token::Type type);
   bool Match(Token::Type type);
-  Token Consume(Token::Type type, const char* error_message);
-  Token Consume(Token::Type* types,
-                size_t num_types,
-                const char* error_message);
-  Token Consume();
+  const Token& Consume(Token::Type type, const char* error_message);
+  const Token& Consume(Token::Type* types,
+                       size_t num_types,
+                       const char* error_message);
+  const Token& Consume();
 
   // Call this only if !at_end().
   const Token& cur_token() const { return tokens_[cur_]; }
@@ -128,6 +128,7 @@ class Parser {
 
   static ParserHelper expressions_[Token::NUM_TYPES];
 
+  Token invalid_token_;
   Err* err_;
 
   // Current index into the tokens.