weblayer,wpt: Update script so that it can mark tests as flaky

Updates wpt_update_expectations.py so that it can mark tests as flaky
by adding pass expectations for tests with unexpected passes.

Bug: 1050754
Change-Id: I071c21d574763b9fa5d7d3f1637ad2236933941f
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2303869
Commit-Queue: Rakib Hasan <rmhasan@google.com>
Reviewed-by: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#790450}

third_party/blink/tools/blinkpy/w3c/android_wpt_expectations_updater_unittest.py

@@ -37,6 +37,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
         'crbug.com/1050754 external/wpt/cat.html [ Failure ]\n'
         'external/wpt/dog.html [ Crash Timeout ]\n'
         'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
+        'external/wpt/unexpected_pass.html [ Failure ]\n'
         '\n'
         '# This comment will not be deleted\n'
         'crbug.com/111111 external/wpt/hello_world.html [ Crash ]\n')
@@ -105,6 +106,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
                         'actual': 'CRASH CRASH TIMEOUT',
                         'is_unexpected': True,
                     },
+                    'unexpected_pass.html': {
+                        'expected': 'FAIL',
+                        'actual': 'PASS',
+                        'is_unexpected': True
+                    },
                     'dog.html': {
                         'expected': 'SKIP',
                         'actual': 'SKIP',
@@ -116,7 +122,8 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
         updater = AndroidWPTExpectationsUpdater(
             host, ['-vvv',  '--android-product', ANDROID_WEBVIEW,
                    '--clean-up-test-expectations',
-                   '--clean-up-affected-tests-only'])
+                   '--clean-up-affected-tests-only',
+                   '--include-unexpected-pass'])
         updater.git_cl = MockGitCL(host, {
             Build('MOCK Android Pie', 123):
             TryJobStatus('COMPLETED', 'FAILURE')})
@@ -137,6 +144,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
              'crbug.com/1050754 external/wpt/ghi.html [ Timeout ]\n'
              'crbug.com/1111111 crbug.com/1050754'
              ' external/wpt/jkl.html [ Failure ]\n'
+             'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
              'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
              'external/wpt/www.html [ Crash Failure ]\n'
              '\n'
@@ -194,6 +202,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
                         'actual': 'CRASH CRASH TIMEOUT',
                         'is_unexpected': True,
                     },
+                    'unexpected_pass.html': {
+                        'expected': 'FAIL',
+                        'actual': 'PASS',
+                        'is_unexpected': True,
+                    },
                     'new.html': {
                         'expected': 'PASS',
                         'actual': 'CRASH CRASH FAIL',
@@ -205,7 +218,8 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
         updater = AndroidWPTExpectationsUpdater(
             host, ['-vvv', '--android-product', ANDROID_WEBLAYER,
                    '--clean-up-test-expectations',
-                   '--clean-up-affected-tests-only'])
+                   '--clean-up-affected-tests-only',
+                   '--include-unexpected-pass'])
         updater.git_cl = MockGitCL(host, {
             Build('MOCK Android Weblayer - Pie', 123):
             TryJobStatus('COMPLETED', 'FAILURE')})
@@ -227,6 +241,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
              'crbug.com/1111111 crbug.com/1050754'
              ' external/wpt/jkl.html [ Failure ]\n'
              'crbug.com/1050754 external/wpt/new.html [ Failure Crash ]\n'
+             'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
              'crbug.com/6789043 external/wpt/van.html [ Failure ]\n'
              'external/wpt/www.html [ Crash Failure ]\n'
              '\n'
@@ -273,6 +288,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
                         'actual': 'SKIP',
                         'is_unexpected': True,
                     },
+                    'unexpected_pass.html': {
+                        'expected': 'FAIL',
+                        'actual': 'PASS',
+                        'is_unexpected': True
+                    },
                 },
             }, step_name=WEBLAYER_WPT_STEP + ' (with patch)'),
             step_name=WEBLAYER_WPT_STEP + ' (with patch)')
@@ -295,6 +315,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
                         'expected': 'SKIP',
                         'actual': 'SKIP',
                     },
+                    'unexpected_pass.html': {
+                        'expected': 'FAIL',
+                        'actual': 'PASS',
+                        'is_unexpected': True
+                    },
                 },
             }, step_name=WEBVIEW_WPT_STEP + ' (with patch)'),
             step_name=WEBVIEW_WPT_STEP + ' (with patch)')
@@ -318,6 +343,11 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
                         'actual': 'SKIP',
                         'is_unexpected': True,
                     },
+                    'unexpected_pass.html': {
+                        'expected': 'FAIL',
+                        'actual': 'PASS',
+                        'is_unexpected': True
+                    },
                 },
             }, step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)'),
             step_name=CHROME_ANDROID_WPT_STEP + ' (with patch)')
@@ -325,6 +355,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
             host, ['-vvv',
                    '--clean-up-test-expectations',
                    '--clean-up-affected-tests-only',
+                   '--include-unexpected-pass',
                    '--android-product', ANDROID_WEBLAYER,
                    '--android-product', CHROME_ANDROID,
                    '--android-product', ANDROID_WEBVIEW])
@@ -360,6 +391,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
              'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
              'external/wpt/dog.html [ Crash Timeout ]\n'
              'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
+             'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
              'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
              'crbug.com/1050754 external/wpt/weblayer_only.html [ Failure Crash ]\n'
              'external/wpt/www.html [ Crash Failure ]\n'
@@ -379,6 +411,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
              'crbug.com/1050754 external/wpt/def.html [ Crash ]\n'
              'external/wpt/dog.html [ Crash Timeout ]\n'
              'crbug.com/1111111 external/wpt/jkl.html [ Failure ]\n'
+             'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
              'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
              'crbug.com/1050754 external/wpt/webview_only.html [ Timeout ]\n'
              'external/wpt/www.html [ Crash Failure ]\n'
@@ -400,6 +433,7 @@ class AndroidWPTExpectationsUpdaterTest(LoggingTestCase):
              'external/wpt/dog.html [ Crash Timeout ]\n'
              'crbug.com/1111111 crbug.com/1050754'
              ' external/wpt/jkl.html [ Failure ]\n'
+             'crbug.com/1050754 external/wpt/unexpected_pass.html [ Failure Pass ]\n'
              'crbug.com/6789043 external/wpt/wagon.html [ Failure ]\n'
              'external/wpt/www.html [ Crash Failure ]\n'
              '\n'

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater.py

@@ -44,7 +44,6 @@ class WPTExpectationsUpdater(object):
         self.git_cl = GitCL(host)
         self.git = self.host.git(self.finder.chromium_base())
         self.configs_with_no_results = []
-        self.configs_with_all_pass = []
         self.patchset = None
 
         # Get options from command line arguments.
@@ -83,6 +82,11 @@ class WPTExpectationsUpdater(object):
         log_level = logging.DEBUG if self.options.verbose else logging.INFO
         configure_logging(logging_level=log_level, include_time=True)
 
+        if not(self.options.android_product or
+                self.options.update_android_expectations_only):
+            assert not self.options.include_unexpected_pass, (
+                'Command line argument --include-unexpected-pass is not '
+                'supported in desktop mode.')
         self.patchset = self.options.patchset
 
         if (self.options.clean_up_test_expectations or
@@ -132,6 +136,12 @@ class WPTExpectationsUpdater(object):
             '--android-product', action='append', default=[],
             help='Android products whose baselines will be updated.',
             choices=PRODUCTS)
+        parser.add_argument(
+            '--include-unexpected-pass',
+            action='store_true',
+            help='Adds Pass to tests with failure expectations. '
+                 'This command line argument can be used to mark tests '
+                 'as flaky.')
 
     def update_expectations(self):
         """Downloads text new baselines and adds test expectations lines.
@@ -153,9 +163,8 @@ class WPTExpectationsUpdater(object):
         # Here we build up a dict of failing test results for all platforms.
         test_expectations = {}
         for build, job_status in build_to_status.iteritems():
-            if job_status.result == 'SUCCESS':
-                self.configs_with_all_pass.extend(
-                    self.get_builder_configs(build))
+            if (job_status.result == 'SUCCESS' and
+                    not self.options.include_unexpected_pass):
                 continue
             result_dicts = self.get_failing_results_dicts(build)
             for result_dict in result_dicts:
@@ -236,13 +245,13 @@ class WPTExpectationsUpdater(object):
             self.configs_with_no_results.extend(self.get_builder_configs(build))
             return []
 
-        failing_test_results = []
+        unexpected_test_results = []
         for results_set in test_results_list:
             results_dict = self.generate_failing_results_dict(
                 build, results_set)
             if results_dict:
-                failing_test_results.append(results_dict)
-        return failing_test_results
+                unexpected_test_results.append(results_dict)
+        return unexpected_test_results
 
     def _get_web_test_results(self, build):
         """Gets web tests results for a builder.
@@ -291,9 +300,6 @@ class WPTExpectationsUpdater(object):
             raise ScriptError('No configuration was found for builder and web test'
                               ' step combination ')
         config = configs[0]
-        if config in self.configs_with_all_pass:
-            return {}
-
         for result in web_test_results.didnt_run_as_expected_results():
             # TODO(rmhasan) If a test fails unexpectedly then it runs multiple
             # times until, it passes or a retry limit is reached. Even though
@@ -301,7 +307,11 @@ class WPTExpectationsUpdater(object):
             # creating test expectations for. Maybe we should add a mode
             # which creates expectations for tests that are flaky but still
             # pass in a web test step.
-            if result.did_pass():
+
+            # Create flaky expectations for flaky tests on Android. In order to
+            # do this we should add 'Pass' to all tests with failing
+            # expectations that pass in the patchset's try job.
+            if result.did_pass() and not self.options.include_unexpected_pass:
                 continue
 
             test_name = result.test_name()

third_party/blink/tools/blinkpy/w3c/wpt_expectations_updater_unittest.py

@@ -157,6 +157,17 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             'crbug.com/626703 [ Mac10.10 ] external/wpt/test/path.html [ Timeout ]\n'
         )
 
+    def test_cmd_arg_include_unexpected_pass_raieses_exception(self):
+        host = self.mock_host()
+        expectations_path = \
+            host.port_factory.get().path_to_generic_test_expectations_file()
+        host.filesystem.write_text_file(expectations_path,
+                                        WPTExpectationsUpdater.MARKER_COMMENT + '\n')
+        updater = WPTExpectationsUpdater(host, args=['--include-unexpected-pass'])
+        with self.assertRaises(AssertionError) as ctx:
+            updater.run()
+        self.assertIn('--include-unexpected-pass', str(ctx.exception))
+
     def test_get_failing_results_dict_only_passing_results(self):
         host = self.mock_host()
         host.results_fetcher.set_results(
@@ -1039,8 +1050,6 @@ class WPTExpectationsUpdaterTest(LoggingTestCase):
             }
         }
         tests_to_rebaseline, _ = updater.get_tests_to_rebaseline(two)
-        # external/wpt/test/zzzz.html is another possible candidate, but it
-        # is not listed in the results dict, so it shall not be rebaselined.
         self.assertEqual(tests_to_rebaseline, ['external/wpt/test/path.html'])
 
     def test_get_test_to_rebaseline_does_not_return_ref_tests(self):