Skip to content

GitLab

T
tangled
Commit 99171a94 authored by yoz@chromium.org's avatar yoz@chromium.org
Browse files
Options

Add global presubmit that JSON and IDL files can be parsed. 

Sometimes, changes to JSON/IDL files make them invalid. It's easier to check these at presubmit time than discovering they can't be parsed at runtime.

This just moves the check from chrome/common/extensions/api to top-level.
This presubmit check excludes files in test/data directories.

BUG=366395

Review URL: https://codereview.chromium.org/239283008

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@274432 0039d316-1c4b-4281-b951-d872f2087c98
parent 21075b5d
Hide whitespace changes
Inline Side-by-side
Showing with 325 additions and 344 deletions
PRESUBMIT.py
View file @ 99171a94
......@@ -1063,6 +1063,106 @@ def _CheckUserActionUpdate(input_api, output_api):
return []
def _GetJSONParseError(input_api, filename, eat_comments=True):
try:
contents = input_api.ReadFile(filename)
if eat_comments:
json_comment_eater = input_api.os_path.join(
input_api.PresubmitLocalPath(),
'tools', 'json_comment_eater', 'json_comment_eater.py')
process = input_api.subprocess.Popen(
[input_api.python_executable, json_comment_eater],
stdin=input_api.subprocess.PIPE,
stdout=input_api.subprocess.PIPE,
universal_newlines=True)
(contents, _) = process.communicate(input=contents)
input_api.json.loads(contents)
except ValueError as e:
return e
return None
def _GetIDLParseError(input_api, filename):
try:
contents = input_api.ReadFile(filename)
idl_schema = input_api.os_path.join(
input_api.PresubmitLocalPath(),
'tools', 'json_schema_compiler', 'idl_schema.py')
process = input_api.subprocess.Popen(
[input_api.python_executable, idl_schema],
stdin=input_api.subprocess.PIPE,
stdout=input_api.subprocess.PIPE,
stderr=input_api.subprocess.PIPE,
universal_newlines=True)
(_, error) = process.communicate(input=contents)
return error or None
except ValueError as e:
return e
def _CheckParseErrors(input_api, output_api):
"""Check that IDL and JSON files do not contain syntax errors."""
actions = {
'.idl': _GetIDLParseError,
'.json': _GetJSONParseError,
}
# These paths contain test data and other known invalid JSON files.
excluded_patterns = [
'test/data/',
'^components/policy/resources/policy_templates.json$',
]
# Most JSON files are preprocessed and support comments, but these do not.
json_no_comments_patterns = [
'^testing/',
]
# Only run IDL checker on files in these directories.
idl_included_patterns = [
'^chrome/common/extensions/api/',
'^extensions/common/api/',
]
def get_action(affected_file):
filename = affected_file.LocalPath()
return actions.get(input_api.os_path.splitext(filename)[1])
def MatchesFile(patterns, path):
for pattern in patterns:
if input_api.re.search(pattern, path):
return True
return False
def FilterFile(affected_file):
action = get_action(affected_file)
if not action:
return False
path = affected_file.LocalPath()
if MatchesFile(excluded_patterns, path):
return False
if (action == _GetIDLParseError and
not MatchesFile(idl_included_patterns, path)):
return False
return True
results = []
for affected_file in input_api.AffectedFiles(
file_filter=FilterFile, include_deletes=False):
action = get_action(affected_file)
kwargs = {}
if (action == _GetJSONParseError and
MatchesFile(json_no_comments_patterns, affected_file.LocalPath())):
kwargs['eat_comments'] = False
parse_error = action(input_api,
affected_file.AbsoluteLocalPath(),
**kwargs)
if parse_error:
results.append(output_api.PresubmitError('%s could not be parsed: %s' %
(affected_file.LocalPath(), parse_error)))
return results
def _CheckJavaStyle(input_api, output_api):
"""Runs checkstyle on changed java files and returns errors if any exist."""
original_sys_path = sys.path
......@@ -1162,6 +1262,7 @@ def _CommonChecks(input_api, output_api):
results.extend(_CheckCygwinShell(input_api, output_api))
results.extend(_CheckUserActionUpdate(input_api, output_api))
results.extend(_CheckNoDeprecatedCSS(input_api, output_api))
results.extend(_CheckParseErrors(input_api, output_api))
if any('PRESUBMIT.py' == f.LocalPath() for f in input_api.AffectedFiles()):
results.extend(input_api.canned_checks.RunUnitTestsInDirectory(
......
PRESUBMIT_test.py
View file @ 99171a94
......@@ -3,23 +3,43 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import json
import os
import re
import subprocess
import sys
import unittest
import PRESUBMIT
_TEST_DATA_DIR = 'base/test/data/presubmit'
class MockInputApi(object):
def __init__(self):
self.json = json
self.re = re
self.os_path = os.path
self.python_executable = sys.executable
self.subprocess = subprocess
self.files = []
self.is_committing = False
def AffectedFiles(self):
return self.files
def PresubmitLocalPath(self):
return os.path.dirname(__file__)
def ReadFile(self, filename, mode='rU'):
for file_ in self.files:
if file_.LocalPath() == filename:
return '\n'.join(file_.NewContents())
# Otherwise, file is not in our mock API.
raise IOError, "No such file or directory: '%s'" % filename
class MockOutputApi(object):
class PresubmitResult(object):
......@@ -431,5 +451,201 @@ class CheckAddedDepsHaveTetsApprovalsTest(unittest.TestCase):
self.assertEqual(expected, files_to_check);
class JSONParsingTest(unittest.TestCase):
def testSuccess(self):
input_api = MockInputApi()
filename = 'valid_json.json'
contents = ['// This is a comment.',
'{',
' "key1": ["value1", "value2"],',
' "key2": 3 // This is an inline comment.',
'}'
]
input_api.files = [MockFile(filename, contents)]
self.assertEqual(None,
PRESUBMIT._GetJSONParseError(input_api, filename))
def testFailure(self):
input_api = MockInputApi()
test_data = [
('invalid_json_1.json',
['{ x }'],
'Expecting property name: line 1 column 2 (char 2)'),
('invalid_json_2.json',
['// Hello world!',
'{ "hello": "world }'],
'Unterminated string starting at: line 2 column 12 (char 12)'),
('invalid_json_3.json',
['{ "a": "b", "c": "d", }'],
'Expecting property name: line 1 column 22 (char 22)'),
('invalid_json_4.json',
['{ "a": "b" "c": "d" }'],
'Expecting , delimiter: line 1 column 11 (char 11)'),
]
input_api.files = [MockFile(filename, contents)
for (filename, contents, _) in test_data]
for (filename, _, expected_error) in test_data:
actual_error = PRESUBMIT._GetJSONParseError(input_api, filename)
self.assertEqual(expected_error, str(actual_error))
def testNoEatComments(self):
input_api = MockInputApi()
file_with_comments = 'file_with_comments.json'
contents_with_comments = ['// This is a comment.',
'{',
' "key1": ["value1", "value2"],',
' "key2": 3 // This is an inline comment.',
'}'
]
file_without_comments = 'file_without_comments.json'
contents_without_comments = ['{',
' "key1": ["value1", "value2"],',
' "key2": 3',
'}'
]
input_api.files = [MockFile(file_with_comments, contents_with_comments),
MockFile(file_without_comments,
contents_without_comments)]
self.assertEqual('No JSON object could be decoded',
str(PRESUBMIT._GetJSONParseError(input_api,
file_with_comments,
eat_comments=False)))
self.assertEqual(None,
PRESUBMIT._GetJSONParseError(input_api,
file_without_comments,
eat_comments=False))
class IDLParsingTest(unittest.TestCase):
def testSuccess(self):
input_api = MockInputApi()
filename = 'valid_idl_basics.idl'
contents = ['// Tests a valid IDL file.',
'namespace idl_basics {',
' enum EnumType {',
' name1,',
' name2',
' };',
'',
' dictionary MyType1 {',
' DOMString a;',
' };',
'',
' callback Callback1 = void();',
' callback Callback2 = void(long x);',
' callback Callback3 = void(MyType1 arg);',
' callback Callback4 = void(EnumType type);',
'',
' interface Functions {',
' static void function1();',
' static void function2(long x);',
' static void function3(MyType1 arg);',
' static void function4(Callback1 cb);',
' static void function5(Callback2 cb);',
' static void function6(Callback3 cb);',
' static void function7(Callback4 cb);',
' };',
'',
' interface Events {',
' static void onFoo1();',
' static void onFoo2(long x);',
' static void onFoo2(MyType1 arg);',
' static void onFoo3(EnumType type);',
' };',
'};'
]
input_api.files = [MockFile(filename, contents)]
self.assertEqual(None,
PRESUBMIT._GetIDLParseError(input_api, filename))
def testFailure(self):
input_api = MockInputApi()
test_data = [
('invalid_idl_1.idl',
['//',
'namespace test {',
' dictionary {',
' DOMString s;',
' };',
'};'],
'Unexpected "{" after keyword "dictionary".\n'),
# TODO(yoz): Disabled because it causes the IDL parser to hang.
# See crbug.com/363830.
# ('invalid_idl_2.idl',
# (['namespace test {',
# ' dictionary MissingSemicolon {',
# ' DOMString a',
# ' DOMString b;',
# ' };',
# '};'],
# 'Unexpected symbol DOMString after symbol a.'),
('invalid_idl_3.idl',
['//',
'namespace test {',
' enum MissingComma {',
' name1',
' name2',
' };',
'};'],
'Unexpected symbol name2 after symbol name1.'),
('invalid_idl_4.idl',
['//',
'namespace test {',
' enum TrailingComma {',
' name1,',
' name2,',
' };',
'};'],
'Trailing comma in block.'),
('invalid_idl_5.idl',
['//',
'namespace test {',
' callback Callback1 = void(;',
'};'],
'Unexpected ";" after "(".'),
('invalid_idl_6.idl',
['//',
'namespace test {',
' callback Callback1 = void(long );',
'};'],
'Unexpected ")" after symbol long.'),
('invalid_idl_7.idl',
['//',
'namespace test {',
' interace Events {',
' static void onFoo1();',
' };',
'};'],
'Unexpected symbol Events after symbol interace.'),
('invalid_idl_8.idl',
['//',
'namespace test {',
' interface NotEvent {',
' static void onFoo1();',
' };',
'};'],
'Did not process Interface Interface(NotEvent)'),
('invalid_idl_9.idl',
['//',
'namespace test {',
' interface {',
' static void function1();',
' };',
'};'],
'Interface missing name.'),
]
input_api.files = [MockFile(filename, contents)
for (filename, contents, _) in test_data]
for (filename, _, expected_error) in test_data:
actual_error = PRESUBMIT._GetIDLParseError(input_api, filename)
self.assertTrue(expected_error in str(actual_error),
"'%s' not found in '%s'" % (expected_error, actual_error))
if __name__ == '__main__':
unittest.main()
chrome/common/extensions/api/PRESUBMIT.py deleted 100644 → 0
View file @ 21075b5d
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
def _GetJSONParseError(input_api, filename):
try:
contents = input_api.ReadFile(filename)
json_comment_eater = input_api.os_path.join(
input_api.PresubmitLocalPath(),
'..', '..', '..', '..', 'tools',
'json_comment_eater', 'json_comment_eater.py')
process = input_api.subprocess.Popen(
[input_api.python_executable, json_comment_eater],
stdin=input_api.subprocess.PIPE,
stdout=input_api.subprocess.PIPE,
universal_newlines=True)
(nommed, _) = process.communicate(input=contents)
input_api.json.loads(nommed)
except ValueError as e:
return e
return None
def _GetIDLParseError(input_api, filename):
idl_schema = input_api.os_path.join(
input_api.PresubmitLocalPath(),
'..', '..', '..', '..', 'tools',
'json_schema_compiler', 'idl_schema.py')
process = input_api.subprocess.Popen(
[input_api.python_executable, idl_schema, filename],
stdout=input_api.subprocess.PIPE,
stderr=input_api.subprocess.PIPE,
universal_newlines=True)
(_, error) = process.communicate()
return error or None
def _GetParseErrors(input_api, output_api):
# Run unit tests.
results = []
if input_api.AffectedFiles(
file_filter=lambda f: 'PRESUBMIT' in f.LocalPath()):
results = input_api.canned_checks.RunUnitTestsInDirectory(
input_api, output_api, '.', whitelist=[r'^PRESUBMIT_test\.py$'])
actions = {
'.idl': _GetIDLParseError,
'.json': _GetJSONParseError,
}
def get_action(affected_file):
filename = affected_file.LocalPath()
return actions.get(input_api.os_path.splitext(filename)[1])
for affected_file in input_api.AffectedFiles(
file_filter=
lambda f: "test_presubmit" not in f.LocalPath() and get_action(f),
include_deletes=False):
parse_error = get_action(affected_file)(input_api,
affected_file.AbsoluteLocalPath())
if parse_error:
results.append(output_api.PresubmitError('%s could not be parsed: %s' %
(affected_file.LocalPath(), parse_error)))
return results
def CheckChangeOnUpload(input_api, output_api):
return _GetParseErrors(input_api, output_api)
def CheckChangeOnCommit(input_api, output_api):
return _GetParseErrors(input_api, output_api)
chrome/common/extensions/api/PRESUBMIT_test.py deleted 100755 → 0
View file @ 21075b5d
#!/usr/bin/env python
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import json
import os
import subprocess
import sys
import unittest
import PRESUBMIT
class MockInputApi(object):
def __init__(self):
self.json = json
self.os_path = os.path
self.subprocess = subprocess
self.python_executable = sys.executable
def PresubmitLocalPath(self):
return os.path.dirname(__file__)
def ReadFile(self, filename, mode='rU'):
with open(filename, mode=mode) as f:
return f.read()
class JSONParsingTest(unittest.TestCase):
def testSuccess(self):
input_api = MockInputApi()
filename = 'test_presubmit/valid_json.json'
self.assertEqual(None,
PRESUBMIT._GetJSONParseError(input_api, filename))
def testFailure(self):
input_api = MockInputApi()
expected_errors = [
'Expecting property name: line 8 column 3 (char 9)',
'Invalid control character at: line 8 column 19 (char 25)',
'Expecting property name: line 8 column 23 (char 29)',
'Expecting , delimiter: line 8 column 12 (char 18)',
]
actual_errors = [
str(PRESUBMIT._GetJSONParseError(input_api, filename))
for filename in sorted(glob.glob('test_presubmit/invalid_*.json'))
]
self.assertEqual(expected_errors, actual_errors)
class IDLParsingTest(unittest.TestCase):
def testSuccess(self):
input_api = MockInputApi()
filename = 'test_presubmit/valid_idl_basics.idl'
self.assertEqual(None,
PRESUBMIT._GetIDLParseError(input_api, filename))
def testFailure(self):
input_api = MockInputApi()
expected_errors = [
'Unexpected "{" after keyword "dictionary".',
'Unexpected symbol DOMString after symbol a.',
'Unexpected symbol name2 after symbol name1.',
'Trailing comma in block.',
'Unexpected ";" after "(".',
'Unexpected ")" after symbol long.',
'Unexpected symbol Events after symbol interace.',
'Did not process Interface Interface(NotEvent)',
'Interface missing name.',
]
actual_errors = [
PRESUBMIT._GetIDLParseError(input_api, filename)
for filename in sorted(glob.glob('test_presubmit/invalid_*.idl'))
]
for (expected_error, actual_error) in zip(expected_errors, actual_errors):
self.assertTrue(expected_error in actual_error)
if __name__ == "__main__":
unittest.main()
chrome/common/extensions/api/test_presubmit/invalid_idl_1.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Unexpected "{" after keyword "dictionary".
dictionary {
DOMString s;
};
};
chrome/common/extensions/api/test_presubmit/invalid_idl_2.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Unexpected symbol DOMString after symbol a.
dictionary MissingSemicolon {
DOMString a
DOMString b;
};
};
chrome/common/extensions/api/test_presubmit/invalid_idl_3.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Unexpected symbol name2 after symbol name1.
enum MissingComma {
name1
name2
};
};
chrome/common/extensions/api/test_presubmit/invalid_idl_4.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Trailing comma in block.
enum TrailingComma {
name1,
name2,
};
};
chrome/common/extensions/api/test_presubmit/invalid_idl_5.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Unexpected ";" after "(".
callback Callback1 = void(;
};
chrome/common/extensions/api/test_presubmit/invalid_idl_6.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Unexpected ")" after symbol long.
callback Callback1 = void(long );
};
chrome/common/extensions/api/test_presubmit/invalid_idl_7.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Unexpected symbol Events after symbol interace.
interace Events {
static void onFoo1();
};
};
chrome/common/extensions/api/test_presubmit/invalid_idl_8.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Did not process Interface Interface(NotEvent).
interface NotEvent {
static void onFoo1();
};
};
chrome/common/extensions/api/test_presubmit/invalid_idl_9.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid IDL file.
namespace test {
// Interface missing name.
interface {
static void function1();
};
};
chrome/common/extensions/api/test_presubmit/invalid_json_1.json deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid JSON file.
// Expecting property name: line 8 column 3 (char 9).
{ x }
chrome/common/extensions/api/test_presubmit/invalid_json_2.json deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid JSON file.
// Invalid control character at: line 8 column 19 (char 25).
{ "hello": "world }
chrome/common/extensions/api/test_presubmit/invalid_json_3.json deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid JSON file.
// Expecting property name: line 8 column 23 (char 29).
{ "a": "b", "c": "d", }
chrome/common/extensions/api/test_presubmit/invalid_json_4.json deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests an invalid JSON file.
// Expecting , delimiter: line 8 column 12 (char 18).
{ "a": "b" "c": "d" }
chrome/common/extensions/api/test_presubmit/valid_idl_basics.idl deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests a valid IDL file.
namespace idl_basics {
enum EnumType {
name1,
name2
};
dictionary MyType1 {
DOMString a;
};
callback Callback1 = void();
callback Callback2 = void(long x);
callback Callback3 = void(MyType1 arg);
callback Callback4 = void(EnumType type);
interface Functions {
static void function1();
static void function2(long x);
static void function3(MyType1 arg);
static void function4(Callback1 cb);
static void function5(Callback2 cb);
static void function6(Callback3 cb);
static void function7(Callback4 cb);
};
interface Events {
static void onFoo1();
static void onFoo2(long x);
static void onFoo2(MyType1 arg);
static void onFoo3(EnumType type);
};
};
chrome/common/extensions/api/test_presubmit/valid_json.json deleted 100644 → 0
View file @ 21075b5d
// Copyright 2013 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Tests a valid IDL file.
// This is a comment.
{
"key1": ["value1", "value2"],
"key2": 3 // This is an inline comment.
}
tools/json_schema_compiler/idl_schema.py
View file @ 99171a94
......@@ -469,8 +469,14 @@ def Main():
Dump a json serialization of parse result for the IDL files whose names
were passed in on the command line.
'''
for filename in sys.argv[1:]:
schema = Load(filename)
if len(sys.argv) > 1:
for filename in sys.argv[1:]:
schema = Load(filename)
print json.dumps(schema, indent=2)
else:
contents = sys.stdin.read()
idl = idl_parser.IDLParser().ParseData(contents, '<stdin>')
schema = IDLSchema(idl).process()
print json.dumps(schema, indent=2)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment