Unrevert 244233 "Add support for using expressions as enum values."

First time broke the build with a non-ASCII character in a comment.

BUG=320090
TBR=darin@chromium.org

Review URL: https://codereview.chromium.org/131833004

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@244250 0039d316-1c4b-4281-b951-d872f2087c98

mojo/public/bindings/parse/mojo_lexer.py

+# PLY based Lexer class, based on pycparser by Eli Bendersky.
+#
+# Copyright (c) 2012, Eli Bendersky
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above copyright notice,
+#   this list of conditions and the following disclaimer in the documentation
+#   and/or other materials provided with the distribution.
+# * Neither the name of Eli Bendersky nor the names of its contributors may
+#   be used to endorse or promote products derived from this software without
+#   specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
+# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import re
+import sys
+import os.path
+
+# Try to load the ply module, if not, then assume it is in the third_party
+# directory.
+try:
+  # Disable lint check which fails to find the ply module.
+  # pylint: disable=F0401
+  from ply.lex import TOKEN
+except ImportError:
+  module_path, module_name = os.path.split(__file__)
+  third_party = os.path.join(
+      module_path, os.pardir, os.pardir, os.pardir, os.pardir, 'third_party')
+  sys.path.append(third_party)
+  # pylint: disable=F0401
+  from ply.lex import TOKEN
+
+
+class Lexer(object):
+  ######################--   PRIVATE   --######################
+
+  ##
+  ## Internal auxiliary methods
+  ##
+  def _error(self, msg, token):
+    print('%s at line %d' % (msg, token.lineno))
+    self.lexer.skip(1)
+
+  ##
+  ## Reserved keywords
+  ##
+  keywords = (
+    'HANDLE',
+    'DATA_PIPE_CONSUMER',
+    'DATA_PIPE_PRODUCER',
+    'MESSAGE_PIPE',
+
+    'MODULE',
+    'STRUCT',
+    'INTERFACE',
+    'ENUM',
+    'VOID',
+  )
+
+  keyword_map = {}
+  for keyword in keywords:
+    keyword_map[keyword.lower()] = keyword
+
+  ##
+  ## All the tokens recognized by the lexer
+  ##
+  tokens = keywords + (
+    # Identifiers
+    'NAME',
+
+    # constants
+    'ORDINAL',
+    'INT_CONST_DEC', 'INT_CONST_OCT', 'INT_CONST_HEX',
+    'FLOAT_CONST', 'HEX_FLOAT_CONST',
+    'CHAR_CONST',
+    'WCHAR_CONST',
+
+    # String literals
+    'STRING_LITERAL',
+    'WSTRING_LITERAL',
+
+    # Operators
+    'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD',
+    'OR', 'AND', 'NOT', 'XOR', 'LSHIFT', 'RSHIFT',
+    'LOR', 'LAND', 'LNOT',
+    'LT', 'LE', 'GT', 'GE', 'EQ', 'NE',
+
+    # Assignment
+    'EQUALS',
+
+    # Conditional operator (?)
+    'CONDOP',
+
+    # Delimeters
+    'LPAREN', 'RPAREN',         # ( )
+    'LBRACKET', 'RBRACKET',     # [ ]
+    'LBRACE', 'RBRACE',         # { }
+    'SEMI', 'COLON',            # ; :
+    'COMMA',                    # .
+  )
+
+  ##
+  ## Regexes for use in tokens
+  ##
+  ##
+
+  # valid C identifiers (K&R2: A.2.3), plus '$' (supported by some compilers)
+  identifier = r'[a-zA-Z_$][0-9a-zA-Z_$]*'
+
+  hex_prefix = '0[xX]'
+  hex_digits = '[0-9a-fA-F]+'
+
+  # integer constants (K&R2: A.2.5.1)
+  integer_suffix_opt = \
+      r'(([uU]ll)|([uU]LL)|(ll[uU]?)|(LL[uU]?)|([uU][lL])|([lL][uU]?)|[uU])?'
+  decimal_constant = \
+      '(0'+integer_suffix_opt+')|([1-9][0-9]*'+integer_suffix_opt+')'
+  octal_constant = '0[0-7]*'+integer_suffix_opt
+  hex_constant = hex_prefix+hex_digits+integer_suffix_opt
+
+  bad_octal_constant = '0[0-7]*[89]'
+
+  # character constants (K&R2: A.2.5.2)
+  # Note: a-zA-Z and '.-~^_!=&;,' are allowed as escape chars to support #line
+  # directives with Windows paths as filenames (..\..\dir\file)
+  # For the same reason, decimal_escape allows all digit sequences. We want to
+  # parse all correct code, even if it means to sometimes parse incorrect
+  # code.
+  #
+  simple_escape = r"""([a-zA-Z._~!=&\^\-\\?'"])"""
+  decimal_escape = r"""(\d+)"""
+  hex_escape = r"""(x[0-9a-fA-F]+)"""
+  bad_escape = r"""([\\][^a-zA-Z._~^!=&\^\-\\?'"x0-7])"""
+
+  escape_sequence = \
+      r"""(\\("""+simple_escape+'|'+decimal_escape+'|'+hex_escape+'))'
+  cconst_char = r"""([^'\\\n]|"""+escape_sequence+')'
+  char_const = "'"+cconst_char+"'"
+  wchar_const = 'L'+char_const
+  unmatched_quote = "('"+cconst_char+"*\\n)|('"+cconst_char+"*$)"
+  bad_char_const = \
+      r"""('"""+cconst_char+"""[^'\n]+')|('')|('"""+ \
+      bad_escape+r"""[^'\n]*')"""
+
+  # string literals (K&R2: A.2.6)
+  string_char = r"""([^"\\\n]|"""+escape_sequence+')'
+  string_literal = '"'+string_char+'*"'
+  wstring_literal = 'L'+string_literal
+  bad_string_literal = '"'+string_char+'*'+bad_escape+string_char+'*"'
+
+  # floating constants (K&R2: A.2.5.3)
+  exponent_part = r"""([eE][-+]?[0-9]+)"""
+  fractional_constant = r"""([0-9]*\.[0-9]+)|([0-9]+\.)"""
+  floating_constant = \
+      '(((('+fractional_constant+')'+ \
+      exponent_part+'?)|([0-9]+'+exponent_part+'))[FfLl]?)'
+  binary_exponent_part = r'''([pP][+-]?[0-9]+)'''
+  hex_fractional_constant = \
+      '((('+hex_digits+r""")?\."""+hex_digits+')|('+hex_digits+r"""\.))"""
+  hex_floating_constant = \
+      '('+hex_prefix+'('+hex_digits+'|'+hex_fractional_constant+')'+ \
+      binary_exponent_part+'[FfLl]?)'
+
+  ##
+  ## Rules for the normal state
+  ##
+  t_ignore = ' \t'
+
+  # Newlines
+  def t_NEWLINE(self, t):
+    r'\n+'
+    t.lexer.lineno += t.value.count("\n")
+
+  # Operators
+  t_PLUS              = r'\+'
+  t_MINUS             = r'-'
+  t_TIMES             = r'\*'
+  t_DIVIDE            = r'/'
+  t_MOD               = r'%'
+  t_OR                = r'\|'
+  t_AND               = r'&'
+  t_NOT               = r'~'
+  t_XOR               = r'\^'
+  t_LSHIFT            = r'<<'
+  t_RSHIFT            = r'>>'
+  t_LOR               = r'\|\|'
+  t_LAND              = r'&&'
+  t_LNOT              = r'!'
+  t_LT                = r'<'
+  t_GT                = r'>'
+  t_LE                = r'<='
+  t_GE                = r'>='
+  t_EQ                = r'=='
+  t_NE                = r'!='
+
+  # =
+  t_EQUALS            = r'='
+
+  # ?
+  t_CONDOP            = r'\?'
+
+  # Delimeters
+  t_LPAREN            = r'\('
+  t_RPAREN            = r'\)'
+  t_LBRACKET          = r'\['
+  t_RBRACKET          = r'\]'
+  t_LBRACE            = r'\{'
+  t_RBRACE            = r'\}'
+  t_COMMA             = r','
+  t_SEMI              = r';'
+  t_COLON             = r':'
+
+  t_STRING_LITERAL    = string_literal
+  t_ORDINAL           = r'@[0-9]*'
+
+  # The following floating and integer constants are defined as
+  # functions to impose a strict order (otherwise, decimal
+  # is placed before the others because its regex is longer,
+  # and this is bad)
+  #
+  @TOKEN(floating_constant)
+  def t_FLOAT_CONST(self, t):
+    return t
+
+  @TOKEN(hex_floating_constant)
+  def t_HEX_FLOAT_CONST(self, t):
+    return t
+
+  @TOKEN(hex_constant)
+  def t_INT_CONST_HEX(self, t):
+    return t
+
+  @TOKEN(bad_octal_constant)
+  def t_BAD_CONST_OCT(self, t):
+    msg = "Invalid octal constant"
+    self._error(msg, t)
+
+  @TOKEN(octal_constant)
+  def t_INT_CONST_OCT(self, t):
+    return t
+
+  @TOKEN(decimal_constant)
+  def t_INT_CONST_DEC(self, t):
+    return t
+
+  # Must come before bad_char_const, to prevent it from
+  # catching valid char constants as invalid
+  #
+  @TOKEN(char_const)
+  def t_CHAR_CONST(self, t):
+    return t
+
+  @TOKEN(wchar_const)
+  def t_WCHAR_CONST(self, t):
+    return t
+
+  @TOKEN(unmatched_quote)
+  def t_UNMATCHED_QUOTE(self, t):
+    msg = "Unmatched '"
+    self._error(msg, t)
+
+  @TOKEN(bad_char_const)
+  def t_BAD_CHAR_CONST(self, t):
+    msg = "Invalid char constant %s" % t.value
+    self._error(msg, t)
+
+  @TOKEN(wstring_literal)
+  def t_WSTRING_LITERAL(self, t):
+    return t
+
+  # unmatched string literals are caught by the preprocessor
+
+  @TOKEN(bad_string_literal)
+  def t_BAD_STRING_LITERAL(self, t):
+    msg = "String contains invalid escape code"
+    self._error(msg, t)
+
+  @TOKEN(identifier)
+  def t_NAME(self, t):
+    t.type = self.keyword_map.get(t.value, "NAME")
+    return t
+
+  # Ignore C and C++ style comments
+  def t_COMMENT(self, t):
+    r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)'
+    pass
+
+  def t_error(self, t):
+    msg = 'Illegal character %s' % repr(t.value[0])
+    self._error(msg, t)

mojo/public/bindings/parse/mojo_parser.py

@@ -9,7 +9,6 @@
 import sys
 import os.path
 
-
 # Try to load the ply module, if not, then assume it is in the third_party
 # directory.
 try:
@@ -26,6 +25,8 @@ except ImportError:
   from ply import lex
   from ply import yacc
 
+from mojo_lexer import Lexer
+
 
 def ListFromConcat(*items):
   """Generate list by concatenating inputs"""
@@ -41,114 +42,13 @@ def ListFromConcat(*items):
   return itemsout
 
 
-class Lexer(object):
-
-  # This field is required by lex to specify the complete list of valid tokens.
-  tokens = (
-    'NAME',
-    'NUMBER',
-
-    'ORDINAL',
-
-    'HANDLE',
-    'DATAPIPECONSUMER',
-    'DATAPIPEPRODUCER',
-    'MESSAGEPIPE',
-
-    'MODULE',
-    'STRUCT',
-    'INTERFACE',
-    'ENUM',
-    'VOID',
-
-    'LCURLY',
-    'RCURLY',
-    'LPAREN',
-    'RPAREN',
-    'LANGLE',
-    'RANGLE',
-    'LBRACKET',
-    'RBRACKET',
-    'COMMA',
-    'SEMICOLON',
-    'EQUALS',
-  )
-
-  t_LCURLY     = r'{'
-  t_RCURLY     = r'}'
-  t_LPAREN     = r'\('
-  t_RPAREN     = r'\)'
-  t_LANGLE     = r'<'
-  t_RANGLE     = r'>'
-  t_LBRACKET   = r'\['
-  t_RBRACKET   = r'\]'
-  t_COMMA      = r','
-  t_SEMICOLON  = r';'
-  t_EQUALS     = r'='
-  t_NAME       = r'[a-zA-Z_][a-zA-Z0-9_]*'
-  t_NUMBER     = r'\d+'
-  t_ORDINAL    = r'@[0-9]*'
-
-  def t_HANDLE(self, t):
-    r'handle'
-    return t
-
-  def t_DATAPIPECONSUMER(self, t):
-    r'data_pipe_consumer'
-    return t
-
-  def t_DATAPIPEPRODUCER(self, t):
-    r'data_pipe_producer'
-    return t
-
-  def t_MESSAGEPIPE(self, t):
-    r'message_pipe'
-    return t
-
-  def t_MODULE(self, t):
-    r'module'
-    return t
-
-  def t_STRUCT(self, t):
-    r'struct'
-    return t
-
-  def t_INTERFACE(self, t):
-    r'interface'
-    return t
-
-  def t_ENUM(self, t):
-    r'enum'
-    return t
-
-  def t_VOID(self, t):
-    r'void'
-    return t
-
-  # Ignore C and C++ style comments
-  def t_COMMENT(self, t):
-    r'(/\*(.|\n)*?\*/)|(//.*(\n[ \t]*//.*)*)'
-    pass
-
-  # Ignored characters
-  t_ignore = " \t"
-
-  def t_newline(self, t):
-    r'\n+'
-    t.lexer.lineno += t.value.count("\n")
-
-  def t_error(self, t):
-    print("Illegal character '%s'" % t.value[0])
-    t.lexer.skip(1)
-
-
 class Parser(object):
 
   def __init__(self, lexer):
     self.tokens = lexer.tokens
 
   def p_module(self, p):
-    """module : MODULE NAME LCURLY definitions RCURLY"""
+    """module : MODULE NAME LBRACE definitions RBRACE"""
     p[0] = ('MODULE', p[2], p[4])
 
   def p_definitions(self, p):
@@ -179,12 +79,12 @@ class Parser(object):
       p[0] = ListFromConcat(p[1], p[3])
 
   def p_attribute(self, p):
-    """attribute : NAME EQUALS NUMBER
+    """attribute : NAME EQUALS expression
                  | NAME EQUALS NAME"""
     p[0] = ('ATTRIBUTE', p[1], p[3])
 
   def p_struct(self, p):
-    """struct : attribute_section STRUCT NAME LCURLY struct_body RCURLY SEMICOLON"""
+    """struct : attribute_section STRUCT NAME LBRACE struct_body RBRACE SEMI"""
     p[0] = ('STRUCT', p[3], p[1], p[5])
 
   def p_struct_body(self, p):
@@ -195,11 +95,11 @@ class Parser(object):
       p[0] = ListFromConcat(p[1], p[2])
 
   def p_field(self, p):
-    """field : typename NAME ordinal SEMICOLON"""
+    """field : typename NAME ordinal SEMI"""
     p[0] = ('FIELD', p[1], p[2], p[3])
 
   def p_interface(self, p):
-    """interface : attribute_section INTERFACE NAME LCURLY interface_body RCURLY SEMICOLON"""
+    """interface : attribute_section INTERFACE NAME LBRACE interface_body RBRACE SEMI"""
     p[0] = ('INTERFACE', p[3], p[1], p[5])
 
   def p_interface_body(self, p):
@@ -210,7 +110,7 @@ class Parser(object):
       p[0] = ListFromConcat(p[1], p[2])
 
   def p_method(self, p):
-    """method : VOID NAME LPAREN parameters RPAREN ordinal SEMICOLON"""
+    """method : VOID NAME LPAREN parameters RPAREN ordinal SEMI"""
     p[0] = ('METHOD', p[2], p[4], p[6])
 
   def p_parameters(self, p):
@@ -240,13 +140,13 @@ class Parser(object):
     p[0] = p[1]
 
   def p_specializedhandle(self, p):
-    """specializedhandle : HANDLE LANGLE specializedhandlename RANGLE"""
+    """specializedhandle : HANDLE LT specializedhandlename GT"""
     p[0] = "handle<" + p[3] + ">"
 
   def p_specializedhandlename(self, p):
-    """specializedhandlename : DATAPIPECONSUMER
-                             | DATAPIPEPRODUCER
-                             | MESSAGEPIPE"""
+    """specializedhandlename : DATA_PIPE_CONSUMER
+                             | DATA_PIPE_PRODUCER
+                             | MESSAGE_PIPE"""
     p[0] = p[1]
 
   def p_array(self, p):
@@ -260,7 +160,7 @@ class Parser(object):
       p[0] = p[1]
 
   def p_enum(self, p):
-    """enum : ENUM NAME LCURLY enum_fields RCURLY SEMICOLON"""
+    """enum : ENUM NAME LBRACE enum_fields RBRACE SEMI"""
     p[0] = ('ENUM', p[2], p[4])
 
   def p_enum_fields(self, p):
@@ -274,12 +174,84 @@ class Parser(object):
 
   def p_enum_field(self, p):
     """enum_field : NAME
-                  | NAME EQUALS NUMBER"""
+                  | NAME EQUALS expression"""
     if len(p) == 2:
       p[0] = ('ENUM_FIELD', p[1], None)
     else:
       p[0] = ('ENUM_FIELD', p[1], p[3])
 
+  ### Expressions ###
+
+  def p_expression(self, p):
+    """expression : conditional_expression"""
+    p[0] = p[1]
+
+  def p_conditional_expression(self, p):
+    """conditional_expression : binary_expression
+                              | binary_expression CONDOP expression COLON conditional_expression"""
+    # Just pass the arguments through. I don't think it's possible to preserve
+    # the spaces of the original, so just put a single space between them.
+    p[0] = ' '.join(p[1:])
+
+  # PLY lets us specify precedence of operators, but since we don't actually
+  # evaluate them, we don't need that here.
+  def p_binary_expression(self, p):
+    """binary_expression : unary_expression
+                         | binary_expression binary_operator binary_expression"""
+    p[0] = ' '.join(p[1:])
+
+  def p_binary_operator(self, p):
+    """binary_operator : TIMES
+                       | DIVIDE
+                       | MOD
+                       | PLUS
+                       | MINUS
+                       | RSHIFT
+                       | LSHIFT
+                       | LT
+                       | LE
+                       | GE
+                       | GT
+                       | EQ
+                       | NE
+                       | AND
+                       | OR
+                       | XOR
+                       | LAND
+                       | LOR"""
+    p[0] = p[1]
+
+  def p_unary_expression(self, p):
+    """unary_expression : primary_expression
+                        | unary_operator expression"""
+    p[0] = ''.join(p[1:])
+
+  def p_unary_operator(self, p):
+    """unary_operator : TIMES
+                      | PLUS
+                      | MINUS
+                      | NOT
+                      | LNOT"""
+    p[0] = p[1]
+
+  def p_primary_expression(self, p):
+    """primary_expression : constant
+                          | NAME
+                          | LPAREN expression RPAREN"""
+    p[0] = ''.join(p[1:])
+
+  def p_constant(self, p):
+    """constant : INT_CONST_DEC
+                | INT_CONST_OCT
+                | INT_CONST_HEX
+                | FLOAT_CONST
+                | HEX_FLOAT_CONST
+                | CHAR_CONST
+                | WCHAR_CONST
+                | STRING_LITERAL
+                | WSTRING_LITERAL"""
+    p[0] = ''.join(p[1:])
+
   def p_error(self, e):
     print('error: %s'%e)
 

mojo/public/bindings/sample/sample_service.mojom

@@ -5,8 +5,9 @@
 module sample {
 
 enum BarType {
-  BAR_VERTICAL = 1,
-  BAR_HORIZONTAL = 2,
+  BAR_VERTICAL = 1 << 0,
+  BAR_HORIZONTAL = (1 << 1) + 0,
+  BAR_BOTH = BAR_VERTICAL | BAR_HORIZONTAL,
   BAR_INVALID
 };