Remove support of NeedsManualRebaseline in layout test

Bug: 621126
Change-Id: If111cc2b1f980f79362d5e52d101e915ae7ba6ef
Reviewed-on: https://chromium-review.googlesource.com/1140927
Commit-Queue: Robert Ma <robertma@chromium.org>
Commit-Queue: Ned Nguyen <nednguyen@google.com>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#576200}

third_party/blink/tools/blinkpy/web_tests/models/test_expectations.py

@@ -53,7 +53,6 @@ V8_BUG_PREFIX = 'code.google.com/p/v8/issues/detail?id='
 NAMED_BUG_PREFIX = 'Bug('
 
 MISSING_KEYWORD = 'Missing'
-NEEDS_MANUAL_REBASELINE_KEYWORD = 'NeedsManualRebaseline'
 
 
 class ParseError(Exception):
@@ -75,7 +74,6 @@ class TestExpectationParser(object):
     # FIXME: Rename these to *_KEYWORD as in MISSING_KEYWORD above, but make
     # the case studdly-caps to match the actual file contents.
     REBASELINE_MODIFIER = 'rebaseline'
-    NEEDS_MANUAL_REBASELINE_MODIFIER = 'needsmanualrebaseline'
     PASS_EXPECTATION = 'pass'
     SKIP_MODIFIER = 'skip'
     SLOW_MODIFIER = 'slow'
@@ -165,14 +163,6 @@ class TestExpectationParser(object):
         if self.REBASELINE_MODIFIER in expectations:
             expectation_line.warnings.append('REBASELINE should only be used for running rebaseline.py. Cannot be checked in.')
 
-        if self.NEEDS_MANUAL_REBASELINE_MODIFIER in expectations:
-            for test in expectation_line.matching_tests:
-                if self._port.reference_files(test):
-                    text_expected_filename = self._port.expected_filename(test, '.txt')
-                    if not self._port.host.filesystem.exists(text_expected_filename):
-                        expectation_line.warnings.append(
-                            'A reftest without text expectation cannot be marked as NeedsManualRebaseline')
-
         specifiers = [specifier.lower() for specifier in expectation_line.specifiers]
         if self.REBASELINE_MODIFIER in expectations and ('debug' in specifiers or 'release' in specifiers):
             expectation_line.warnings.append('A test cannot be rebaselined for Debug/Release.')
@@ -307,7 +297,6 @@ class TestExpectationLine(object):
         MISSING_KEYWORD: 'MISSING',
         'Pass': 'PASS',
         'Rebaseline': 'REBASELINE',
-        NEEDS_MANUAL_REBASELINE_KEYWORD: 'NEEDSMANUALREBASELINE',
         'Skip': 'SKIP',
         'Slow': 'SLOW',
         'Timeout': 'TIMEOUT',
@@ -449,7 +438,7 @@ class TestExpectationLine(object):
         if 'MISSING' in expectations:
             warnings.append(
                 '"Missing" expectations are not allowed; download new baselines '
-                '(see https://goo.gl/SHVYrZ), or as a fallback, use "NeedsManualRebaseline".')
+                '(see https://goo.gl/SHVYrZ), or as a fallback, use "SKIP".')
 
         expectation_line.bugs = bugs
         expectation_line.specifiers = specifiers
@@ -913,7 +902,6 @@ class TestExpectations(object):
         'leak': LEAK,
         'missing': MISSING,
         TestExpectationParser.SKIP_MODIFIER: SKIP,
-        TestExpectationParser.NEEDS_MANUAL_REBASELINE_MODIFIER: NEEDS_MANUAL_REBASELINE,
         TestExpectationParser.WONTFIX_MODIFIER: WONTFIX,
         TestExpectationParser.SLOW_MODIFIER: SLOW,
         TestExpectationParser.REBASELINE_MODIFIER: REBASELINE,

third_party/blink/tools/blinkpy/web_tests/models/test_expectations_unittest.py

@@ -54,19 +54,15 @@ class Base(unittest.TestCase):
         return ['failures/expected/text.html',
                 'failures/expected/image_checksum.html',
                 'failures/expected/crash.html',
-                'failures/expected/needsmanualrebaseline.html',
                 'failures/expected/image.html',
                 'failures/expected/timeout.html',
                 'passes/text.html',
-                'reftests/failures/expected/needsmanualrebaseline.html',
-                'reftests/failures/expected/needsmanualrebaseline_with_txt.html',
                 'reftests/failures/expected/has_unused_expectation.html']
 
     def get_basic_expectations(self):
         return """
 Bug(test) failures/expected/text.html [ Failure ]
 Bug(test) failures/expected/crash.html [ Crash ]
-Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
 Bug(test) failures/expected/image_checksum.html [ Crash ]
 Bug(test) failures/expected/image.html [ Crash Mac ]
 """
@@ -196,49 +192,6 @@ class MiscTests(Base):
         test_set = self._exp.get_test_set(CRASH)
         self.assertEqual(test_set, set(['failures/expected/crash.html', 'failures/expected/image_checksum.html']))
 
-    def test_needs_rebaseline_reftest(self):
-        try:
-            filesystem = self._port.host.filesystem
-            filesystem.write_text_file(
-                filesystem.join(
-                    self._port.layout_tests_dir(),
-                    'reftests/failures/expected/needsmanualrebaseline.html'),
-                'content')
-            filesystem.write_text_file(
-                filesystem.join(
-                    self._port.layout_tests_dir(),
-                    'reftests/failures/expected/needsmanualrebaseline-expected.html'),
-                'content')
-            filesystem.write_text_file(
-                filesystem.join(
-                    self._port.layout_tests_dir(),
-                    'reftests/failures/expected/needsmanualrebaseline_with_txt.html'),
-                'content')
-            filesystem.write_text_file(
-                filesystem.join(
-                    self._port.layout_tests_dir(),
-                    'reftests/failures/expected/needsmanualrebaseline_with_txt.html'),
-                'content')
-            filesystem.write_text_file(
-                filesystem.join(
-                    self._port.layout_tests_dir(),
-                    'reftests/failures/expected/needsmanualrebaseline_with_txt-expected.html'),
-                'content')
-            filesystem.write_text_file(
-                filesystem.join(
-                    self._port.layout_tests_dir(),
-                    'reftests/failures/expected/needsmanualrebaseline_with_txt-expected.txt'),
-                'content')
-            self.parse_exp(
-                'Bug(user) reftests/failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]\n'
-                'Bug(user) reftests/failures/expected/needsmanualrebaseline_with_txt.html [ NeedsManualRebaseline ]\n',
-                is_lint_mode=True)
-            self.assertFalse(True, "ParseError wasn't raised")
-        except ParseError as error:
-            warnings = ('expectations:1 A reftest without text expectation cannot be marked as '
-                        'NeedsManualRebaseline reftests/failures/expected/needsmanualrebaseline.html')
-            self.assertEqual(str(error), warnings)
-
     def test_parse_warning(self):
         try:
             filesystem = self._port.host.filesystem
@@ -308,8 +261,6 @@ class MiscTests(Base):
         self.assertFalse(match('failures/expected/image_checksum.html', PASS, True))
         self.assertFalse(match('failures/expected/image_checksum.html', PASS, False))
         self.assertFalse(match('failures/expected/crash.html', PASS, False))
-        self.assertTrue(match('failures/expected/needsmanualrebaseline.html', TEXT, True))
-        self.assertFalse(match('failures/expected/needsmanualrebaseline.html', CRASH, True))
         self.assertTrue(match('passes/text.html', PASS, False))
 
     def test_sanitizer_flag(self):
@@ -630,25 +581,6 @@ Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 Bug(y) [ Win Mac Debug ] failures/expected/foo.html [ Crash ]
 """, actual_expectations)
 
-    def test_remove_needs_manual_rebaseline(self):
-        host = MockHost()
-        test_port = host.port_factory.get('test-win-win7', None)
-        test_port.test_exists = lambda test: True
-        test_port.test_isfile = lambda test: True
-
-        test_config = test_port.test_configuration()
-        test_port.expectations_dict = lambda: {
-            'expectations': 'Bug(x) [ Win ] failures/expected/foo.html [ NeedsManualRebaseline ]\n'
-        }
-        expectations = TestExpectations(test_port, self.get_basic_tests())
-
-        actual_expectations = expectations.remove_configurations([('failures/expected/foo.html', test_config)])
-
-        self.assertEqual(
-            'Bug(x) [ Win7 Debug ] failures/expected/foo.html [ NeedsManualRebaseline ]\n'
-            'Bug(x) [ Win10 ] failures/expected/foo.html [ NeedsManualRebaseline ]\n',
-            actual_expectations)
-
     def test_remove_multiple_configurations(self):
         host = MockHost()
         test_port = host.port_factory.get('test-win-win7', None)

third_party/blink/tools/blinkpy/web_tests/models/test_run_results_unittest.py

@@ -204,7 +204,6 @@ class SummarizedResultsTest(unittest.TestCase):
                 'MISSING': 0,
                 'TEXT': 1,
                 'IMAGE': 1,
-                'NEEDSMANUALREBASELINE': 0,
                 'PASS': 0,
                 'REBASELINE': 0,
                 'SKIP': 0,
@@ -225,7 +224,6 @@ class SummarizedResultsTest(unittest.TestCase):
                 'MISSING': 0,
                 'TEXT': 0,
                 'IMAGE': 0,
-                'NEEDSMANUALREBASELINE': 0,
                 'PASS': 1,
                 'REBASELINE': 0,
                 'SKIP': 0,
@@ -246,7 +244,6 @@ class SummarizedResultsTest(unittest.TestCase):
                 'MISSING': 0,
                 'TEXT': 0,
                 'IMAGE': 0,
-                'NEEDSMANUALREBASELINE': 0,
                 'PASS': 5,
                 'REBASELINE': 0,
                 'SKIP': 1,

third_party/blink/tools/blinkpy/web_tests/port/test.py

@@ -110,9 +110,9 @@ class TestList(object):
 #
 # These numbers may need to be updated whenever we add or delete tests. This includes virtual tests.
 #
-TOTAL_TESTS = 135
+TOTAL_TESTS = 133
 TOTAL_WONTFIX = 3
-TOTAL_SKIPS = 21 + TOTAL_WONTFIX
+TOTAL_SKIPS = 20 + TOTAL_WONTFIX
 TOTAL_CRASHES = 76
 
 UNEXPECTED_PASSES = 1
@@ -127,7 +127,6 @@ def unit_test_list():
     tests.add('failures/expected/device_failure.html', device_failure=True)
     tests.add('failures/expected/timeout.html', timeout=True)
     tests.add('failures/expected/leak.html', leak=True)
-    tests.add('failures/expected/needsmanualrebaseline.html', actual_text='needsmanualrebaseline text')
     tests.add('failures/expected/image.html',
               actual_image='image_fail-pngtEXtchecksum\x00checksum_fail',
               expected_image='image-pngtEXtchecksum\x00checksum-png')
@@ -306,7 +305,6 @@ Bug(test) failures/expected/image_checksum.html [ Failure ]
 Bug(test) failures/expected/keyboard.html [ Crash ]
 Bug(test) failures/expected/leak.html [ Leak ]
 Bug(test) failures/expected/mismatch.html [ Failure ]
-Bug(test) failures/expected/needsmanualrebaseline.html [ NeedsManualRebaseline ]
 Bug(test) failures/expected/newlines_leading.html [ Failure ]
 Bug(test) failures/expected/newlines_trailing.html [ Failure ]
 Bug(test) failures/expected/newlines_with_excess_CR.html [ Failure ]

third_party/blink/tools/blinkpy/web_tests/update_expectations_unittest.py

@@ -173,8 +173,7 @@ class UpdateTestExpectationsTest(LoggingTestCase):
             Bug(test) test/a.html [ Pass ]
             Bug(test) test/b.html [ Timeout ]
             Bug(test) test/c.html [ Failure Timeout ]
-            Bug(test) test/d.html [ Rebaseline ]
-            Bug(test) test/e.html [ NeedsManualRebaseline ]"""
+            Bug(test) test/d.html [ Rebaseline ]"""
 
         self._expectations_remover = (
             self._create_expectations_remover(self.FLAKE_TYPE))
@@ -215,8 +214,7 @@ class UpdateTestExpectationsTest(LoggingTestCase):
             Bug(test) test/a.html [ Pass ]
             Bug(test) test/b.html [ Failure Pass ]
             Bug(test) test/c.html [ Failure Pass Timeout ]
-            Bug(test) test/d.html [ Rebaseline ]
-            Bug(test) test/e.html [ NeedsManualRebaseline ]"""
+            Bug(test) test/d.html [ Rebaseline ]"""
 
         self._expectations_remover = (
             self._create_expectations_remover(self.FAIL_TYPE))
@@ -352,9 +350,7 @@ class UpdateTestExpectationsTest(LoggingTestCase):
             # Even though the results show all passing, none of the
             # expectations are flaky or failing so we shouldn't remove any.
             Bug(test) test/a.html [ Failure Pass Rebaseline ]
-            Bug(test) test/b.html [ Failure NeedsManualRebaseline Pass ]
-            Bug(test) test/c.html [ Failure Rebaseline ]
-            Bug(test) test/d.html [ Failure NeedsManualRebaseline ]"""
+            Bug(test) test/b.html [ Failure Rebaseline ]"""
 
         self._expectations_remover = self._create_expectations_remover()
         self._define_builders({
@@ -371,8 +367,6 @@ class UpdateTestExpectationsTest(LoggingTestCase):
             'WebKit Linux Trusty': {
                 'test/a.html': ['PASS', 'PASS'],
                 'test/b.html': ['PASS', 'PASS'],
-                'test/c.html': ['PASS', 'PASS'],
-                'test/d.html': ['PASS', 'PASS'],
             }
         }
         updated_expectations = (