Add checks for READMEs for each virtual test suite.

By convention, each virtual test suite should have a README.txt documenting
what directory it mirrors and what command line args are needed, so that
you can tell what an "empty" directory under LayoutTests/ is for.

This patch modifies lint-test-expectations to check that those READMEs
exist, and will help avoid the possibility that we'll delete whole directories
thinking they are unneeded and introduce weird errors when TestExpectations
still refer to nonexistent directories or files.

R=ojan@chromium.org
BUG=369385

Review URL: https://codereview.chromium.org/267803007

git-svn-id: svn://svn.chromium.org/blink/trunk@175748 bbb929c8-8fbe-4397-9dbb-9b2b20218538

third_party/WebKit/LayoutTests/TestExpectations

@@ -1066,6 +1066,10 @@ crbug.com/380151 fast/events/only-valid-drop-targets-receive-file-drop.html [ Fa
 
 crbug.com/380649 [ Mac ] fast/dom/HTMLScriptElement/dont-load-unknown-type.html [ Failure ]
 
+# We only want to run one of the web-animations-api tests in stable mode.
+crbug.com/368946 virtual/stable/web-animations-api [ Skip ]
+crbug.com/368946 virtual/stable/web-animations-api/eased-keyframes.html [ Pass ]
+
 # Temporarily skipped until Chromium side change lands.
 crbug.com/308768 http/tests/security/XFrameOptions/x-frame-options-deny-meta-tag-in-body.html [ Skip ]
 crbug.com/308768 http/tests/security/XFrameOptions/x-frame-options-deny-meta-tag-parent-same-origin-deny.html [ Skip ]

third_party/WebKit/LayoutTests/virtual/android/fullscreen/README.txt

+# This suite runs the tests in fullscreen with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/antialiasedtext/fast/text/README.txt

+# This suite runs the tests in LayoutTests/fast/text/ with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/deferred/inspector/timeline/README.txt

+# This suite runs the tests in LayoutTests/inspector/timeline/ with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/implsidepainting/inspector/timeline/README.txt

+# This suite runs the tests in inspector/timeline with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/linux-subpixel/platform/linux/fast/text/subpixel/README.txt

+# This suite runs the tests in platform/linux/fast/text/subpixel with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/mac-antialiasedtext/fast/text/README.txt

+# This suite runs the tests in fast/text with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/serviceworker/http/tests/serviceworker/README.txt

+# This suite runs the tests in http/tests/serviceworker with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/stable/animations-unprefixed/README.txt

+# This suite runs some of the animations tests with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/stable/fast/css3-text/css3-text-decoration/stable/README.txt

+# This suite runs the tests in fast/css3-css3-text-decoration/stable with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/stable/media/stable/README.txt

+# This suite runs the tests in media/stable with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/stable/web-animations-api/README.txt

+# This suite runs the web-animations-api/eased-keyframes.html with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.
+

third_party/WebKit/LayoutTests/virtual/stable/webexposed/README.txt

+# This suite runs the tests in LayoutTests/webexposed/ with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/targetedstylerecalc/fast/css/invalidation/README.txt

+# This suite runs the tests in fast/css/invalidation with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/threaded/animations/README.txt

+# This suite runs the tests in LayoutTests/animations/ with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/threaded/transitions/README.txt

+# This suite runs the tests in LayoutTests/transitions/ with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/LayoutTests/virtual/windows-directwrite/fast/text/README.txt

+# This suite runs the tests in fast/text with additional flags.
+# See the virtual_test_suites() method in Tools/Scripts/webkitpy/layout_tests/port/base.py.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/lint_test_expectations.py

@@ -46,46 +46,77 @@ EXCEPTIONAL_EXIT_STATUS = 254
 _log = logging.getLogger(__name__)
 
 
-def lint(host, options, logging_stream):
+def lint(host, options):
+    # FIXME: Remove this when we remove the --chromium flag (crbug.com/245504).
+    if options.platform == 'chromium':
+        options.platform = None
+
+    ports_to_lint = [host.port_factory.get(name) for name in host.port_factory.all_port_names(options.platform)]
+    files_linted = set()
+    lint_failed = False
+
+    for port_to_lint in ports_to_lint:
+        expectations_dict = port_to_lint.expectations_dict()
+
+        for expectations_file in expectations_dict.keys():
+            if expectations_file in files_linted:
+                continue
+
+            try:
+                test_expectations.TestExpectations(port_to_lint,
+                    expectations_dict={expectations_file: expectations_dict[expectations_file]},
+                    is_lint_mode=True)
+            except test_expectations.ParseError as e:
+                lint_failed = True
+                _log.error('')
+                for warning in e.warnings:
+                    _log.error(warning)
+                _log.error('')
+            files_linted.add(expectations_file)
+    return lint_failed
+
+
+def check_virtual_test_suites(host, options):
+    port = host.port_factory.get(options=options)
+    fs = host.filesystem
+    layout_tests_dir = port.layout_tests_dir()
+    virtual_suites = port.virtual_test_suites()
+
+    check_failed = False
+    for suite in virtual_suites:
+        comps = [layout_tests_dir] + suite.name.split('/') + ['README.txt']
+        path_to_readme = fs.join(*comps)
+        if not fs.exists(path_to_readme):
+            _log.error('LayoutTests/%s/README.txt is missing (each virtual suite must have one).' % suite.name)
+            check_failed = True
+    if check_failed:
+        _log.error('')
+    return check_failed
+
+
+def set_up_logging(logging_stream):
     logger = logging.getLogger()
     logger.setLevel(logging.INFO)
     handler = logging.StreamHandler(logging_stream)
     logger.addHandler(handler)
+    return (logger, handler)
 
+
+def tear_down_logging(logger, handler):
+    logger.removeHandler(handler)
+
+
+def run_checks(host, options, logging_stream):
+    logger, handler = set_up_logging(logging_stream)
     try:
-        # FIXME: Remove this when we remove the --chromium flag (crbug.com/245504).
-        if options.platform == 'chromium':
-            options.platform = None
-
-        ports_to_lint = [host.port_factory.get(name) for name in host.port_factory.all_port_names(options.platform)]
-        files_linted = set()
-        lint_failed = False
-
-        for port_to_lint in ports_to_lint:
-            expectations_dict = port_to_lint.expectations_dict()
-
-            for expectations_file in expectations_dict.keys():
-                if expectations_file in files_linted:
-                    continue
-
-                try:
-                    test_expectations.TestExpectations(port_to_lint,
-                        expectations_dict={expectations_file: expectations_dict[expectations_file]},
-                        is_lint_mode=True)
-                except test_expectations.ParseError as e:
-                    lint_failed = True
-                    _log.error('')
-                    for warning in e.warnings:
-                        _log.error(warning)
-                    _log.error('')
-                files_linted.add(expectations_file)
-
-        if lint_failed:
+        lint_failed = lint(host, options)
+        check_failed = check_virtual_test_suites(host, options)
+        if lint_failed or check_failed:
             _log.error('Lint failed.')
-            return -1
-
-        _log.info('Lint succeeded.')
-        return 0
+            return 1
+        else:
+            _log.info('Lint succeeded.')
+            return 0
     finally:
         logger.removeHandler(handler)
 
@@ -104,7 +135,7 @@ def main(argv, _, stderr):
         host = Host()
 
     try:
-        exit_status = lint(host, options, stderr)
+        exit_status = run_checks(host, options, stderr)
     except KeyboardInterrupt:
         exit_status = INTERRUPTED_EXIT_STATUS
     except Exception as e:

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/lint_test_expectations_unittest.py

@@ -72,7 +72,7 @@ class FakeFactory(object):
         for port in ports:
             self.ports[port.name] = port
 
-    def get(self, port_name, *args, **kwargs):  # pylint: disable=W0613,E0202
+    def get(self, port_name='a', *args, **kwargs):  # pylint: disable=W0613,E0202
         return self.ports[port_name]
 
     def all_port_names(self, platform=None):  # pylint: disable=W0613,E0202
@@ -89,7 +89,11 @@ class LintTest(unittest.TestCase):
 
         logging_stream = StringIO.StringIO()
         options = optparse.Values({'platform': None})
-        res = lint_test_expectations.lint(host, options, logging_stream)
+        logger, handler = lint_test_expectations.set_up_logging(logging_stream)
+        try:
+            res = lint_test_expectations.lint(host, options)
+        finally:
+            lint_test_expectations.tear_down_logging(logger, handler)
         self.assertEqual(res, 0)
         self.assertEqual(host.ports_parsed, ['a', 'b', 'b-win'])
 
@@ -102,10 +106,13 @@ class LintTest(unittest.TestCase):
         # FIXME: incorrect complaints about spacing pylint: disable=C0322
         host.port_factory.all_port_names = lambda platform=None: [platform]
 
-        res = lint_test_expectations.lint(host, options, logging_stream)
+        logger, handler = lint_test_expectations.set_up_logging(logging_stream)
+        try:
+            res = lint_test_expectations.lint(host, options)
+            self.assertEqual(res, 0)
+        finally:
+            lint_test_expectations.tear_down_logging(logger, handler)
 
-        self.assertEqual(res, 0)
-        self.assertIn('Lint succeeded', logging_stream.getvalue())
 
     def test_lint_test_files__errors(self):
         options = optparse.Values({'platform': 'test', 'debug_rwt_logging': False})
@@ -119,42 +126,75 @@ class LintTest(unittest.TestCase):
         host.port_factory.all_port_names = lambda platform=None: [port.name()]
 
         logging_stream = StringIO.StringIO()
+        logger, handler = lint_test_expectations.set_up_logging(logging_stream)
+        try:
+            res = lint_test_expectations.lint(host, options)
+        finally:
+            lint_test_expectations.tear_down_logging(logger, handler)
 
-        res = lint_test_expectations.lint(host, options, logging_stream)
-
-        self.assertEqual(res, -1)
-        self.assertIn('Lint failed', logging_stream.getvalue())
+        self.assertTrue(res)
         self.assertIn('foo:1', logging_stream.getvalue())
         self.assertIn('bar:1', logging_stream.getvalue())
 
 
+class CheckVirtualSuiteTest(unittest.TestCase):
+    def test_check_virtual_test_suites(self):
+        host = MockHost()
+        options = optparse.Values({'platform': 'test', 'debug_rwt_logging': False})
+        orig_get = host.port_factory.get
+        host.port_factory.get = lambda options: orig_get('test', options=options)
+
+        logging_stream = StringIO.StringIO()
+        logger, handler = lint_test_expectations.set_up_logging(logging_stream)
+        try:
+            res = lint_test_expectations.check_virtual_test_suites(host, options)
+            self.assertTrue(res)
+
+            host.filesystem.exists = lambda path: True
+            res = lint_test_expectations.check_virtual_test_suites(host, options)
+            self.assertFalse(res)
+        finally:
+            lint_test_expectations.tear_down_logging(logger, handler)
+
+
 class MainTest(unittest.TestCase):
-    def test_success(self):
-        orig_lint_fn = lint_test_expectations.lint
+    # unused args pylint: disable=W0613
 
-        # unused args pylint: disable=W0613
-        def interrupting_lint(host, options, logging_stream):
-            raise KeyboardInterrupt
+    def setUp(self):
+        self.orig_lint_fn = lint_test_expectations.lint
+        self.orig_check_fn = lint_test_expectations.check_virtual_test_suites
+        lint_test_expectations.check_virtual_test_suites = lambda host, options: False
 
-        def successful_lint(host, options, logging_stream):
-            return 0
+        self.stdout = StringIO.StringIO()
+        self.stderr = StringIO.StringIO()
 
-        def exception_raising_lint(host, options, logging_stream):
-            assert False
+    def tearDown(self):
+        lint_test_expectations.lint = self.orig_lint_fn
+        lint_test_expectations.check_virtual_test_suites = self.orig_check_fn
 
-        stdout = StringIO.StringIO()
-        stderr = StringIO.StringIO()
-        try:
-            lint_test_expectations.lint = interrupting_lint
-            res = lint_test_expectations.main([], stdout, stderr)
-            self.assertEqual(res, lint_test_expectations.INTERRUPTED_EXIT_STATUS)
+    def test_success(self):
+        lint_test_expectations.lint = lambda host, options: False
+        res = lint_test_expectations.main(['--platform', 'test'], self.stdout, self.stderr)
+        self.assertTrue('Lint succeeded' in self.stderr.getvalue())
+        self.assertEqual(res, 0)
 
-            lint_test_expectations.lint = successful_lint
-            res = lint_test_expectations.main(['--platform', 'test'], stdout, stderr)
-            self.assertEqual(res, 0)
+    def test_failure(self):
+        lint_test_expectations.lint = lambda host, options: True
+        res = lint_test_expectations.main(['--platform', 'test'], self.stdout, self.stderr)
+        self.assertTrue('Lint failed' in self.stderr.getvalue())
+        self.assertEqual(res, 1)
 
-            lint_test_expectations.lint = exception_raising_lint
-            res = lint_test_expectations.main([], stdout, stderr)
-            self.assertEqual(res, lint_test_expectations.EXCEPTIONAL_EXIT_STATUS)
-        finally:
-            lint_test_expectations.lint = orig_lint_fn
+    def test_interrupt(self):
+        def interrupting_lint(host, options):
+            raise KeyboardInterrupt
+
+        lint_test_expectations.lint = interrupting_lint
+        res = lint_test_expectations.main([], self.stdout, self.stderr)
+        self.assertEqual(res, lint_test_expectations.INTERRUPTED_EXIT_STATUS)
+
+    def test_exception(self):
+        def exception_raising_lint(host, options):
+            assert False
+        lint_test_expectations.lint = exception_raising_lint
+        res = lint_test_expectations.main([], self.stdout, self.stderr)
+        self.assertEqual(res, lint_test_expectations.EXCEPTIONAL_EXIT_STATUS)

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

@@ -1734,7 +1734,7 @@ class Port(object):
                              ['--stable-release-mode',
                               '--force-compositing-mode']),
             VirtualTestSuite('stable',
-                             'web-animations-api/eased-keyframes.html',
+                             'web-animations-api',
                              ['--stable-release-mode',
                               '--force-compositing-mode']),
             VirtualTestSuite('linux-subpixel',