Move manual tests from TextExpectation file to WontFix file.

Bug: 738489
Change-Id: I0251f741bee780b3cbb88b13a29b67238b8c0cf5
Reviewed-on: https://chromium-review.googlesource.com/c/1269759
Commit-Queue: Kyle Ju <kyleju@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#607680}

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater.py

@@ -268,9 +268,10 @@ class WPTExpectationsUpdater(object):
         # expectation is correct.
         # We also want to skip any new manual tests that are not automated;
         # see crbug.com/708241 for context.
-        if ('MISSING' in actual_results or
-                '-manual.' in test_name and 'TIMEOUT' in actual_results):
+        if 'MISSING' in actual_results:
             return {'Skip'}
+        if '-manual.' in test_name and 'TIMEOUT' in actual_results:
+            return {'WontFix'}
         expectations = set()
         failure_types = {'TEXT', 'IMAGE+TEXT', 'IMAGE', 'AUDIO'}
         other_types = {'TIMEOUT', 'CRASH', 'PASS'}
@@ -338,11 +339,16 @@ class WPTExpectationsUpdater(object):
 
         specifier_part = self.specifier_part(test_name, port_names)
 
-        line_parts = [result.bug]
+        line_parts = []
         if specifier_part:
             line_parts.append(specifier_part)
         line_parts.append(test_name)
-        line_parts.append('[ %s ]' % ' '.join(self.get_expectations(result, test_name)))
+        expectations = '[ %s ]' % ' '.join(self.get_expectations(result, test_name))
+        line_parts.append(expectations)
+
+        # Only add the bug link if the expectations do not include WontFix.
+        if 'WontFix' not in expectations:
+            line_parts.insert(0, result.bug)
 
         return ' '.join(line_parts)
 
@@ -447,32 +453,51 @@ class WPTExpectationsUpdater(object):
         comment line. If this marker comment line is not found, then everything
         including the marker line is appended to the end of the file.
 
+        All WontFix tests are inserted to NeverFixTests file instead of TextExpectations
+        file.
+
         Args:
             line_dict: A dictionary from test names to a list of test expectation lines.
         """
         if not line_dict:
-            _log.info('No lines to write to TestExpectations.')
+            _log.info('No lines to write to TestExpectations nor NeverFixTests.')
             return
 
-        _log.info('Lines to write to TestExpectations:')
         line_list = []
+        wont_fix_list = []
         for lines in line_dict.itervalues():
             for line in lines:
-                line_list.append(line)
-                _log.info('  %s', line)
-
-        expectations_file_path = self.port.path_to_generic_test_expectations_file()
-        file_contents = self.host.filesystem.read_text_file(expectations_file_path)
-
-        marker_comment_index = file_contents.find(MARKER_COMMENT)
-        if marker_comment_index == -1:
-            file_contents += '\n%s\n' % MARKER_COMMENT
-            file_contents += '\n'.join(line_list)
-        else:
-            end_of_marker_line = (file_contents[marker_comment_index:].find('\n')) + marker_comment_index
-            file_contents = file_contents[:end_of_marker_line + 1] + '\n'.join(line_list) + file_contents[end_of_marker_line:]
-
-        self.host.filesystem.write_text_file(expectations_file_path, file_contents)
+                if 'WontFix' in line:
+                    wont_fix_list.append(line)
+                else:
+                    line_list.append(line)
+
+        if line_list:
+            _log.info('Lines to write to TestExpectations:\n %s', '\n'.join(line_list))
+            # Writes to TestExpectations file.
+            expectations_file_path = self.port.path_to_generic_test_expectations_file()
+            file_contents = self.host.filesystem.read_text_file(expectations_file_path)
+
+            marker_comment_index = file_contents.find(MARKER_COMMENT)
+            if marker_comment_index == -1:
+                file_contents += '\n%s\n' % MARKER_COMMENT
+                file_contents += '\n'.join(line_list)
+            else:
+                end_of_marker_line = (file_contents[marker_comment_index:].find('\n')) + marker_comment_index
+                file_contents = file_contents[:end_of_marker_line + 1] + '\n'.join(line_list) + file_contents[end_of_marker_line:]
+
+            self.host.filesystem.write_text_file(expectations_file_path, file_contents)
+
+        if wont_fix_list:
+            _log.info('Lines to write to NeverFixTests:\n %s', '\n'.join(wont_fix_list))
+            # Writes to NeverFixTests file.
+            wont_fix_path = self.port.path_to_never_fix_tests_file()
+            wont_fix_file_content = self.host.filesystem.read_text_file(wont_fix_path)
+            if not wont_fix_file_content.endswith('\n'):
+                wont_fix_file_content += '\n'
+            wont_fix_file_content += '\n'.join(wont_fix_list)
+            wont_fix_file_content += '\n'
+            self.host.filesystem.write_text_file(wont_fix_path, wont_fix_file_content)
 
     # TODO(robertma): Unit test this method.
     def download_text_baselines(self, test_results):

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater_unittest.py

@@ -291,7 +291,7 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             {'Skip'})
         self.assertEqual(
             updater.get_expectations(SimpleTestResult('Pass', 'TIMEOUT', 'bug'), test_name='foo/bar-manual.html'),
-            {'Skip'})
+            {'WontFix'})
 
     def test_create_line_dict_old_tests(self):
         # In this example, there are two failures that are not in wpt.
@@ -330,6 +330,25 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
                 ],
             })
 
+    def test_create_line_dict_with_manual_tests(self):
+        # In this example, there are three unexpected results for wpt tests.
+        # The new test expectation lines are sorted by test, and then specifier.
+        updater = WPTExpectationsUpdater(self.mock_host())
+        results = {
+            'virtual/foo/external/wpt/test/aa-manual.html': {
+                'test-linux-trusty': SimpleTestResult(expected='PASS', actual='TIMEOUT', bug='crbug.com/test'),
+                'test-mac-mac10.11': SimpleTestResult(expected='FAIL', actual='TIMEOUT', bug='crbug.com/test'),
+            },
+        }
+        self.assertEqual(
+            updater.create_line_dict(results),
+            {
+                'virtual/foo/external/wpt/test/aa-manual.html': [
+                    '[ Trusty ] virtual/foo/external/wpt/test/aa-manual.html [ WontFix ]',
+                    '[ Mac10.11 ] virtual/foo/external/wpt/test/aa-manual.html [ WontFix ]',
+                ],
+            })
+
     def test_specifier_part(self):
         updater = WPTExpectationsUpdater(self.mock_host())
         self.assertEqual(updater.specifier_part('x/y.html', ['test-mac-mac10.10']), '[ Mac10.10 ]')
@@ -473,12 +492,20 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             MARKER_COMMENT + '\n')
         updater = WPTExpectationsUpdater(host)
         line_dict = {'fake/file/path.html': ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']}
+        wontfix_path = host.port_factory.get().path_to_never_fix_tests_file()
+        wontfix_value_origin = host.filesystem.read_text_file(wontfix_path)
+
         updater.write_to_test_expectations(line_dict)
         value = host.filesystem.read_text_file(expectations_path)
+
         self.assertMultiLineEqual(
             value,
             (MARKER_COMMENT + '\n'
              'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n'))
+        wontfix_value = host.filesystem.read_text_file(wontfix_path)
+        self.assertMultiLineEqual(
+            wontfix_value,
+            wontfix_value_origin)
 
     def test_write_to_test_expectations_with_no_marker_comment(self):
         host = self.mock_host()
@@ -488,13 +515,21 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             'crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n')
         updater = WPTExpectationsUpdater(host)
         line_dict = {'fake/file/path.html': ['crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]']}
+        wontfix_path = host.port_factory.get().path_to_never_fix_tests_file()
+        wontfix_value_origin = host.filesystem.read_text_file(wontfix_path)
+
         updater.write_to_test_expectations(line_dict)
         value = host.filesystem.read_text_file(expectations_path)
+
         self.assertMultiLineEqual(
             value,
             ('crbug.com/111 [ Trusty ] foo/bar.html [ Failure ]\n'
              '\n' + MARKER_COMMENT + '\n'
              'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]'))
+        wontfix_value = host.filesystem.read_text_file(wontfix_path)
+        self.assertMultiLineEqual(
+            wontfix_value,
+            wontfix_value_origin)
 
     def test_write_to_test_expectations_with_marker_and_no_lines(self):
         host = self.mock_host()
@@ -502,12 +537,70 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
         host.filesystem.write_text_file(
             expectations_path,
             MARKER_COMMENT + '\n' + 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
+        wontfix_path = host.port_factory.get().path_to_never_fix_tests_file()
+        wontfix_value_origin = host.filesystem.read_text_file(wontfix_path)
+
         updater = WPTExpectationsUpdater(host)
         updater.write_to_test_expectations({})
+
         value = host.filesystem.read_text_file(expectations_path)
         self.assertMultiLineEqual(
             value,
             MARKER_COMMENT + '\n' + 'crbug.com/123 [ Trusty ] fake/file/path.html [ Pass ]\n')
+        wontfix_value = host.filesystem.read_text_file(wontfix_path)
+        self.assertMultiLineEqual(
+            wontfix_value,
+            wontfix_value_origin)
+
+    def test_write_to_test_expectations_with_manual_tests_and_newline(self):
+        host = self.mock_host()
+        expectations_path = host.port_factory.get().path_to_generic_test_expectations_file()
+        wontfix_path = host.port_factory.get().path_to_never_fix_tests_file()
+        line_dict = {'fake/file/path.html': ['[ Trusty ] fake/file/path.html [ WontFix ]']}
+        host.filesystem.write_text_file(
+            expectations_path,
+            MARKER_COMMENT + '\n')
+        host.filesystem.write_text_file(
+            wontfix_path,
+            '[ Trusty ] fake/file/path.html [ WontFix ]\n')
+        updater = WPTExpectationsUpdater(host)
+
+        updater.write_to_test_expectations(line_dict)
+
+        expectations_value = host.filesystem.read_text_file(expectations_path)
+        wontfix_value = host.filesystem.read_text_file(wontfix_path)
+        self.assertMultiLineEqual(
+            expectations_value,
+            MARKER_COMMENT + '\n')
+        self.assertMultiLineEqual(
+            wontfix_value,
+            '[ Trusty ] fake/file/path.html [ WontFix ]\n'
+            '[ Trusty ] fake/file/path.html [ WontFix ]\n')
+
+    def test_write_to_test_expectations_without_newline(self):
+        host = self.mock_host()
+        expectations_path = host.port_factory.get().path_to_generic_test_expectations_file()
+        wontfix_path = host.port_factory.get().path_to_never_fix_tests_file()
+        line_dict = {'fake/file/path.html': ['[ Trusty ] fake/file/path.html [ WontFix ]']}
+        host.filesystem.write_text_file(
+            expectations_path,
+            MARKER_COMMENT + '\n')
+        host.filesystem.write_text_file(
+            wontfix_path,
+            '[ Trusty ] fake/file/path.html [ WontFix ]')
+        updater = WPTExpectationsUpdater(host)
+
+        updater.write_to_test_expectations(line_dict)
+
+        expectations_value = host.filesystem.read_text_file(expectations_path)
+        wontfix_value = host.filesystem.read_text_file(wontfix_path)
+        self.assertMultiLineEqual(
+            expectations_value,
+            MARKER_COMMENT + '\n')
+        self.assertMultiLineEqual(
+            wontfix_value,
+            '[ Trusty ] fake/file/path.html [ WontFix ]\n'
+            '[ Trusty ] fake/file/path.html [ WontFix ]\n')
 
     def test_is_reference_test_given_testharness_test(self):
         updater = WPTExpectationsUpdater(self.mock_host())