Commit f2f3c921 authored by Kenichi Ishibashi's avatar Kenichi Ishibashi Committed by Commit Bot

IDL parser: Parse special comments as AST nodes

Tests in tools/idl_parser are broken as of [1] because the current IDL
parser discards comments but tests assumed that there are comment nodes
which contain test expectations. This CL is basically a revert of [1]
with following modifications:
- Introduce SpecialComments instead of Comments
- SpecialComments are Javadoc-style comment (/** ... */) and they are
  parsed to AST nodes
- Normal comments (C/C++ style; /* ... */ and // ...) are still ignored

The reason to introduce SpecialComments: If we treat normal comments as
tokens and parse them as AST nodes, we have to add a lot of custom rules
to the parser, which will mess up the parser.

[1] https://chromium-review.googlesource.com/c/544424

BUG=740524

Change-Id: I872b6c19e6b844859bbcaa48c828decda36cb300
Reviewed-on: https://chromium-review.googlesource.com/569940Reviewed-by: default avatarHitoshi Yoshida <peria@chromium.org>
Reviewed-by: default avatarKentaro Hara <haraken@chromium.org>
Reviewed-by: default avatarYuki Shiino <yukishiino@chromium.org>
Commit-Queue: Kenichi Ishibashi <bashi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#486629}
parent 8cf0f37f
......@@ -42,6 +42,7 @@ class IDLLexer(object):
'string',
# Symbol and keywords types
'SPECIAL_COMMENT',
'identifier',
# MultiChar operators
......@@ -142,6 +143,14 @@ class IDLLexer(object):
self.AddLines(t.value.count('\n'))
return t
# A Javadoc style comment: /** xxx */
# Unlike t_COMMENT, this is NOT ignored.
# Also note that this should be defined before t_COMMENT.
def t_SPECIAL_COMMENT(self, t):
r'/\*\*(.|\n)+?\*/'
self.AddLines(t.value.count('\n'))
return t
# A C or C++ style comment: /* xxx */ or //
# This token is ignored.
def t_COMMENT(self, t):
......
......@@ -157,7 +157,7 @@ class IDLNode(object):
self.out.extend(proplist)
if filter_nodes == None:
filter_nodes = ['Comment', 'Copyright']
filter_nodes = ['SpecialComment']
search = DumpTreeSearch(accept_props)
self.Traverse(search, filter_nodes)
......
......@@ -53,6 +53,7 @@ ERROR_REMAP = {
'Unexpected ")" after ",".' : 'Missing argument.',
'Unexpected "}" after ",".' : 'Trailing comma in block.',
'Unexpected "}" after "{".' : 'Unexpected empty block.',
'Unexpected comment after "}".' : 'Unexpected trailing comment.',
'Unexpected "{" after keyword "enum".' : 'Enum missing name.',
'Unexpected "{" after keyword "struct".' : 'Struct missing name.',
'Unexpected "{" after keyword "interface".' : 'Interface missing name.',
......@@ -101,6 +102,8 @@ def TokenTypeName(t):
return 'value %s' % t.value
if t.type == 'string' :
return 'string "%s"' % t.value
if t.type == 'SPECIAL_COMMENT':
return 'comment'
if t.type == t.value:
return '"%s"' % t.value
if t.type == ',':
......@@ -110,6 +113,28 @@ def TokenTypeName(t):
return 'keyword "%s"' % t.value
# TODO(bashi): Consider moving this out of idl_parser.
def ExtractSpecialComment(comment):
if not comment.startswith('/**'):
raise ValueError('Special comment must start with /**')
if not comment.endswith('*/'):
raise ValueError('Special comment must end with */')
# Remove comment markers
lines = []
for line in comment[2:-2].split('\n'):
# Remove characters until start marker for this line '*' if found
# otherwise it will be blank.
offs = line.find('*')
if offs >= 0:
line = line[offs + 1:].rstrip()
else:
# TODO(bashi): We may want to keep |line| as is.
line = ''
lines.append(line)
return '\n'.join(lines)
#
# IDL Parser
#
......@@ -196,7 +221,7 @@ def TokenTypeName(t):
class IDLParser(object):
def p_Definitions(self, p):
"""Definitions : ExtendedAttributeList Definition Definitions
| """
| """
if len(p) > 1:
p[2].AddChildren(p[1])
p[0] = ListFromConcat(p[2], p[3])
......@@ -689,12 +714,21 @@ class IDLParser(object):
"""SetlikeRest : SETLIKE '<' Type '>' ';'"""
p[0] = self.BuildProduction('Setlike', p, 2, p[3])
# This rule has custom additions (i.e. SpecialComments).
def p_ExtendedAttributeList(self, p):
"""ExtendedAttributeList : '[' ExtendedAttribute ExtendedAttributes ']'
"""ExtendedAttributeList : SpecialComments '[' ExtendedAttribute ExtendedAttributes ']'
| '[' ExtendedAttribute ExtendedAttributes ']'
| SpecialComments
| """
if len(p) > 3:
if len(p) > 5:
items = ListFromConcat(p[3], p[4])
attribs = self.BuildProduction('ExtAttributes', p, 2, items)
p[0] = ListFromConcat(p[1], attribs)
elif len(p) > 4:
items = ListFromConcat(p[2], p[3])
p[0] = self.BuildProduction('ExtAttributes', p, 1, items)
elif len(p) > 1:
p[0] = p[1]
# Error recovery for ExtendedAttributeList
def p_ExtendedAttributeListError(self, p):
......@@ -1012,6 +1046,14 @@ class IDLParser(object):
"""RecordType : RECORD '<' error ',' Type '>'"""
p[0] = self.BuildError(p, 'RecordType')
# Blink extension: Treat special comments (/** ... */) as AST nodes to
# annotate other nodes. Currently they are used for testing.
def p_SpecialComments(self, p):
"""SpecialComments : SPECIAL_COMMENT SpecialComments
| """
if len(p) > 1:
p[0] = ListFromConcat(self.BuildSpecialComment(p, 1), p[2])
#
# Parser Errors
#
......@@ -1101,6 +1143,11 @@ class IDLParser(object):
childlist.append(self.BuildAttribute('NAME', p[index]))
return self.BuildProduction(cls, p, index, childlist)
def BuildSpecialComment(self, p, index):
name = ExtractSpecialComment(p[index])
childlist = [self.BuildAttribute('NAME', name)]
return self.BuildProduction('SpecialComment', p, index, childlist)
#
# BuildError
#
......
......@@ -21,7 +21,7 @@ class WebIDLParser(unittest.TestCase):
self.filenames = glob.glob('test_parser/*_web.idl')
def _TestNode(self, node):
comments = node.GetListOf('Comment')
comments = node.GetListOf('SpecialComment')
for comment in comments:
check, value = ParseCommentTest(comment.GetName())
if check == 'BUILD':
......@@ -52,7 +52,7 @@ class WebIDLParser(unittest.TestCase):
self.assertTrue(len(children) > 2, 'Expecting children in %s.' %
filename)
for node in filenode.GetChildren()[2:]:
for node in filenode.GetChildren():
self._TestNode(node)
......
......@@ -30,7 +30,7 @@ symatics should exist. This is an exact match.
*/
/* TREE
/** TREE
*Callback(VoidFunc)
* Type()
* PrimitiveType(void)
......@@ -38,7 +38,7 @@ symatics should exist. This is an exact match.
*/
callback VoidFunc = void();
/* TREE
/** TREE
*Callback(VoidFuncLongErr)
* Type()
* PrimitiveType(void)
......@@ -47,7 +47,7 @@ callback VoidFunc = void();
*/
callback VoidFuncLongErr = void ( long );
/* TREE
/** TREE
*Callback(VoidFuncLong)
* Type()
* PrimitiveType(void)
......@@ -58,7 +58,7 @@ callback VoidFuncLongErr = void ( long );
*/
callback VoidFuncLong = void ( long L1 );
/* TREE
/** TREE
*Callback(VoidFuncLongArray)
* Type()
* PrimitiveType(void)
......@@ -70,7 +70,7 @@ callback VoidFuncLong = void ( long L1 );
*/
callback VoidFuncLongArray = void ( long[] L1 );
/* TREE
/** TREE
*Callback(VoidFuncLongArray5)
* Type()
* PrimitiveType(void)
......@@ -83,7 +83,7 @@ callback VoidFuncLongArray = void ( long[] L1 );
callback VoidFuncLongArray5 = void ( long[5] L1 );
/* TREE
/** TREE
*Callback(VoidFuncLongArray54)
* Type()
* PrimitiveType(void)
......@@ -100,7 +100,7 @@ callback VoidFuncLongArray5 = void ( long[5] L1 );
callback VoidFuncLongArray54 = void ( long[5] L1, long long [4] L2 );
/* TREE
/** TREE
*Callback(VoidFuncLongIdent)
* Type()
* PrimitiveType(void)
......
......@@ -30,26 +30,26 @@ symatics should exist. This is an exact match.
*/
/* TREE
/** TREE
*Dictionary(MyDict)
*/
dictionary MyDict { };
/* TREE
/** TREE
*Dictionary(MyDictInherit)
* Inherit(Foo)
*/
dictionary MyDictInherit : Foo {};
/* TREE
/** TREE
*Dictionary(MyDictPartial)
*/
partial dictionary MyDictPartial { };
/* ERROR Unexpected ":" after identifier "MyDictInherit". */
/** ERROR Unexpected ":" after identifier "MyDictInherit". */
partial dictionary MyDictInherit : Foo {};
/* TREE
/** TREE
*Dictionary(MyDictBig)
* Key(setString)
* Type()
......@@ -69,7 +69,7 @@ dictionary MyDictBig {
long unsetLong;
};
/* TREE
/** TREE
*Dictionary(MyDictRequired)
* Key(setLong)
* Type()
......@@ -79,12 +79,12 @@ dictionary MyDictRequired {
required long setLong;
};
/* ERROR Unexpected "{" after keyword "dictionary". */
/** ERROR Unexpected "{" after keyword "dictionary". */
dictionary {
DOMString? setString = null;
};
/* TREE
/** TREE
*Dictionary(MyDictionaryInvalidOptional)
* Key(mandatory)
* Type()
......@@ -96,12 +96,12 @@ dictionary MyDictionaryInvalidOptional {
sequence<DOMString> optional;
};
/* ERROR Unexpected identifier "NoColon" after identifier "ForParent". */
/** ERROR Unexpected identifier "NoColon" after identifier "ForParent". */
dictionary ForParent NoColon {
DOMString? setString = null;
};
/* TREE
/** TREE
*Dictionary(MyDictNull)
* Key(setString)
* Type()
......@@ -112,7 +112,7 @@ dictionary MyDictNull {
DOMString? setString = null;
};
/* ERROR Unexpected keyword "attribute" after "{". */
/** ERROR Unexpected keyword "attribute" after "{". */
dictionary MyDictUnexpectedAttribute {
attribute DOMString foo = "";
};
......@@ -29,45 +29,45 @@ This comment signals that a tree of nodes matching the BUILD comment
symatics should exist. This is an exact match.
*/
/* TREE
/** TREE
*Enum(MealType1)
* EnumItem(rice)
* EnumItem(noodles)
* EnumItem(other)
*/
enum MealType1 {
/* BUILD EnumItem (rice) */
/** BUILD EnumItem (rice) */
"rice",
/* BUILD EnumItem (noodles) */
/** BUILD EnumItem (noodles) */
"noodles",
/* BUILD EnumItem(other) */
/** BUILD EnumItem(other) */
"other"
};
/* BUILD Error(Enum missing name.) */
/* ERROR Enum missing name. */
/** BUILD Error(Enum missing name.) */
/** ERROR Enum missing name. */
enum {
"rice",
"noodles",
"other"
};
/* TREE
/** TREE
*Enum(MealType2)
* EnumItem(rice)
* EnumItem(noodles)
* EnumItem(other)
*/
enum MealType2 {
/* BUILD EnumItem(rice) */
/** BUILD EnumItem(rice) */
"rice",
/* BUILD EnumItem(noodles) */
/** BUILD EnumItem(noodles) */
"noodles",
/* BUILD EnumItem(other) */
/** BUILD EnumItem(other) */
"other"
};
/* TREE
/** TREE
*Enum(TrailingComma)
* EnumItem(rice)
* EnumItem(noodles)
......@@ -79,45 +79,45 @@ enum TrailingComma {
"other",
};
/* BUILD Error(Unexpected string "noodles" after string "rice".) */
/* ERROR Unexpected string "noodles" after string "rice". */
/** BUILD Error(Unexpected string "noodles" after string "rice".) */
/** ERROR Unexpected string "noodles" after string "rice". */
enum MissingComma {
"rice"
"noodles",
"other"
};
/* BUILD Error(Unexpected "," after ",".) */
/* ERROR Unexpected "," after ",". */
/** BUILD Error(Unexpected "," after ",".) */
/** ERROR Unexpected "," after ",". */
enum ExtraComma {
"rice",
"noodles",
,"other",
};
/* BUILD Error(Unexpected keyword "interface" after "{".) */
/* ERROR Unexpected keyword "interface" after "{". */
/** BUILD Error(Unexpected keyword "interface" after "{".) */
/** ERROR Unexpected keyword "interface" after "{". */
enum ExtraComma {
interface,
"noodles",
,"other",
};
/* BUILD Error(Unexpected identifier "somename" after "{".) */
/* ERROR Unexpected identifier "somename" after "{". */
/** BUILD Error(Unexpected identifier "somename" after "{".) */
/** ERROR Unexpected identifier "somename" after "{". */
enum ExtraComma {
somename,
"noodles",
,"other",
};
/* BUILD Enum(MealType3) */
/** BUILD Enum(MealType3) */
enum MealType3 {
/* BUILD EnumItem(rice) */
/** BUILD EnumItem(rice) */
"rice",
/* BUILD EnumItem(noodles) */
/** BUILD EnumItem(noodles) */
"noodles",
/* BUILD EnumItem(other) */
/** BUILD EnumItem(other) */
"other"
};
......@@ -29,24 +29,24 @@ This comment signals that a tree of nodes matching the BUILD comment
symatics should exist. This is an exact match.
*/
/* BUILD Implements(A) */
/* PROP REFERENCE=B */
/** BUILD Implements(A) */
/** PROP REFERENCE=B */
A implements B;
/* ERROR Unexpected ";" after keyword "implements". */
/** ERROR Unexpected ";" after keyword "implements". */
A implements;
/* BUILD Implements(B) */
/* PROP REFERENCE=C */
/** BUILD Implements(B) */
/** PROP REFERENCE=C */
B implements C;
/* ERROR Unexpected keyword "implements" after "]". */
/** ERROR Unexpected keyword "implements" after "]". */
[foo] implements B;
/* BUILD Implements(D) */
/* PROP REFERENCE=E */
/** BUILD Implements(D) */
/** PROP REFERENCE=E */
D implements E;
/* ERROR Unexpected keyword "implements" after comment. */
/** ERROR Unexpected keyword "implements" after comment. */
implements C;
......@@ -30,26 +30,26 @@ symatics should exist. This is an exact match.
*/
/* TREE
/** TREE
*Interface(MyIFace)
*/
interface MyIFace { };
/* TREE
/** TREE
*Interface(MyIFaceInherit)
* Inherit(Foo)
*/
interface MyIFaceInherit : Foo {};
/* TREE
/** TREE
*Interface(MyIFacePartial)
*/
partial interface MyIFacePartial { };
/* ERROR Unexpected ":" after identifier "MyIFaceInherit". */
/** ERROR Unexpected ":" after identifier "MyIFaceInherit". */
partial interface MyIFaceInherit : Foo {};
/* TREE
/** TREE
*Interface(MyIFaceMissingArgument)
* Operation(foo)
* Arguments()
......@@ -64,14 +64,14 @@ interface MyIFaceMissingArgument {
void foo(DOMString arg, );
};
/* TREE
/** TREE
*Error(Unexpected keyword "double" after keyword "readonly".)
*/
interface MyIFaceMissingAttribute {
readonly double foo;
};
/* TREE
/** TREE
*Interface(MyIFaceContainsUnresolvedConflictDiff)
* Operation(foo)
* Arguments()
......@@ -88,7 +88,7 @@ interface MyIFaceContainsUnresolvedConflictDiff {
>>>>>> theirs
};
/* TREE
/** TREE
*Interface(MyIFaceWrongRecordKeyType)
* Operation(foo)
* Arguments()
......@@ -102,7 +102,7 @@ interface MyIFaceWrongRecordKeyType {
void foo(record<int, ByteString> arg);
};
/* TREE
/** TREE
*Interface(MyIFaceBig)
* Const(setString)
* StringType(DOMString)
......@@ -112,7 +112,7 @@ interface MyIFaceBig {
const DOMString? setString = null;
};
/* TREE
/** TREE
*Interface(MyIfaceEmptySequenceDefalutValue)
* Operation(foo)
* Arguments()
......@@ -129,7 +129,7 @@ interface MyIfaceEmptySequenceDefalutValue {
void foo(optional sequence<DOMString> arg = []);
};
/* TREE
/** TREE
*Interface(MyIfaceWithRecords)
* Operation(foo)
* Arguments()
......@@ -160,7 +160,7 @@ interface MyIfaceWithRecords {
double bar(int arg1, record<ByteString, float> arg2);
};
/* TREE
/** TREE
*Interface(MyIFaceBig2)
* Const(nullValue)
* StringType(DOMString)
......@@ -207,7 +207,7 @@ interface MyIFaceBig2 {
};
/* TREE
/** TREE
*Interface(MyIFaceSpecials)
* Operation(set)
* Arguments()
......@@ -239,7 +239,7 @@ interface MyIFaceSpecials {
long long [5][6] GetFiveSix(SomeType arg);
};
/* TREE
/** TREE
*Interface(MyIFaceStringifiers)
* Stringifier()
* Stringifier()
......@@ -264,7 +264,7 @@ interface MyIFaceStringifiers {
stringifier attribute DOMString stringValue;
};
/* TREE
/** TREE
*Interface(MyExtendedAttributeInterface)
* Operation(method)
* Arguments()
......@@ -283,7 +283,7 @@ interface MyExtendedAttributeInterface {
[Attr, MethodIdentList=(Foo, Bar)] void method();
};
/* TREE
/** TREE
*Interface(MyIfacePromise)
* Operation(method1)
* Arguments()
......@@ -317,7 +317,7 @@ interface MyIfacePromise {
Promise method4();
};
/* TREE
/** TREE
*Interface(MyIfaceIterable)
* Iterable()
* Type()
......@@ -333,7 +333,7 @@ interface MyIfaceIterable {
iterable<double, DOMString>;
};
/* TREE
/** TREE
*Interface(MyIfaceMaplike)
* Maplike()
* Type()
......@@ -351,7 +351,7 @@ interface MyIfaceMaplike {
maplike<double, boolean>;
};
/* TREE
/** TREE
*Interface(MyIfaceSetlike)
* Setlike()
* Type()
......@@ -365,7 +365,7 @@ interface MyIfaceSetlike {
setlike<double>;
};
/* TREE
/** TREE
*Interface(MyIfaceSerializer)
* Serializer()
* Serializer()
......@@ -411,7 +411,7 @@ interface MyIfaceSerializer {
serializer = [name1, name2];
};
/* TREE
/** TREE
*Interface(MyIfaceFrozenArray)
* Attribute(foo)
* Type()
......@@ -423,7 +423,7 @@ interface MyIfaceFrozenArray {
readonly attribute FrozenArray<DOMString> foo;
};
/* TREE
/** TREE
*Interface(MyIfaceUnion)
* Attribute(foo)
* Type()
......
......@@ -30,14 +30,14 @@ symatics should exist. This is an exact match.
*/
/* TREE
/** TREE
*Typedef(MyLong)
* Type()
* PrimitiveType(long)
*/
typedef long MyLong;
/* TREE
/** TREE
*Typedef(MyLong)
* ExtAttributes()
* ExtAttribute(foo)
......@@ -46,7 +46,7 @@ typedef long MyLong;
*/
typedef [foo] long MyLong;
/* TREE
/** TREE
*Typedef(MyLongArray)
* Type()
* PrimitiveType(long)
......@@ -54,7 +54,7 @@ typedef [foo] long MyLong;
*/
typedef long[] MyLongArray;
/* TREE
/** TREE
*Typedef(MyLongSizedArray)
* Type()
* PrimitiveType(long)
......@@ -62,7 +62,7 @@ typedef long[] MyLongArray;
*/
typedef long[4] MyLongSizedArray;
/* TREE
/** TREE
*Typedef(MyLongSizedArrayArray)
* Type()
* PrimitiveType(long)
......@@ -71,7 +71,7 @@ typedef long[4] MyLongSizedArray;
*/
typedef long[4][5] MyLongSizedArrayArray;
/* TREE
/** TREE
*Typedef(MyLongArraySizedArray)
* Type()
* PrimitiveType(long)
......@@ -80,7 +80,7 @@ typedef long[4][5] MyLongSizedArrayArray;
*/
typedef long[][5] MyLongArraySizedArray;
/* TREE
/** TREE
*Typedef(MyTypeFive)
* Type()
* Typeref(MyType)
......@@ -88,7 +88,7 @@ typedef long[][5] MyLongArraySizedArray;
*/
typedef MyType[5] MyTypeFive;
/* TREE
/** TREE
*Typedef(MyTypeUnsizedFive)
* Type()
* Typeref(MyType)
......@@ -97,98 +97,98 @@ typedef MyType[5] MyTypeFive;
*/
typedef MyType[][5] MyTypeUnsizedFive;
/* TREE
/** TREE
*Typedef(MyLongLong)
* Type()
* PrimitiveType(long long)
*/
typedef long long MyLongLong;
/* TREE
/** TREE
*Typedef(MyULong)
* Type()
* PrimitiveType(unsigned long)
*/
typedef unsigned long MyULong;
/* TREE
/** TREE
*Typedef(MyULongLong)
* Type()
* PrimitiveType(unsigned long long)
*/
typedef unsigned long long MyULongLong;
/* TREE
/** TREE
*Typedef(MyString)
* Type()
* StringType(DOMString)
*/
typedef DOMString MyString;
/* TREE
/** TREE
*Typedef(MyObject)
* Type()
* PrimitiveType(object)
*/
typedef object MyObject;
/* TREE
/** TREE
*Typedef(MyDate)
* Type()
* PrimitiveType(Date)
*/
typedef Date MyDate;
/* TREE
/** TREE
*Typedef(MyFloat)
* Type()
* PrimitiveType(float)
*/
typedef float MyFloat;
/* TREE
/** TREE
*Typedef(MyUFloat)
* Type()
* PrimitiveType(float)
*/
typedef unrestricted float MyUFloat;
/* TREE
/** TREE
*Typedef(MyDouble)
* Type()
* PrimitiveType(double)
*/
typedef double MyDouble;
/* TREE
/** TREE
*Typedef(MyUDouble)
* Type()
* PrimitiveType(double)
*/
typedef unrestricted double MyUDouble;
/* TREE
/** TREE
*Typedef(MyBool)
* Type()
* PrimitiveType(boolean)
*/
typedef boolean MyBool;
/* TREE
/** TREE
*Typedef(MyByte)
* Type()
* PrimitiveType(byte)
*/
typedef byte MyByte;
/* TREE
/** TREE
*Typedef(MyOctet)
* Type()
* PrimitiveType(octet)
*/
typedef octet MyOctet;
/* TREE
/** TREE
*Typedef(MyRecord)
* Type()
* Record()
......@@ -198,7 +198,7 @@ typedef octet MyOctet;
*/
typedef record<ByteString, int> MyRecord;
/* TREE
/** TREE
*Typedef(MyInvalidRecord)
* Type()
* Error(Unexpected keyword "double" after "<".)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment