Make update_flaky_expectations.py include lines with no passing result.

Make update_flaky_expectations.py script delete lines that do not
fail in the specified way when they don't include any passing results.

Currently expectations that do not fail in the specified way are
removed only when they include a passing result, e.g. the expectation
  bug(test) fast/test.html [ Crash Pass ]
will be removed if the results on the builders show only Passes and
Timeouts because there is no Crash result.

This patch changes the script to also delete the line if the builders
show only Timeouts (and no Passes). This is in the spirit of the test's
description "Scans the TestExpectations file and uses results from
actual builder bots runs to remove tests that are marked as flaky but
don't fail in the specified way."

Bug: 730704
Change-Id: Ib70b5eb43fb58471ef28c346c9a0f7c334283a13
Reviewed-on: https://chromium-review.googlesource.com/1067525
Commit-Queue: Bugs Nash <bugsnash@chromium.org>
Reviewed-by: Quinten Yearsley <qyearsley@chromium.org>
Cr-Commit-Position: refs/heads/master@{#560901}

third_party/blink/tools/blinkpy/web_tests/update_flaky_expectations.py

@@ -108,8 +108,9 @@ class RemoveFlakesOMatic(object):
         if self._port.test_isdir(test_expectation_line.name):
             return False
 
-        # The line can be deleted if the only expectation on the line that appears in the actual
-        # results is the PASS expectation.
+        # The line can be deleted if none of the expectations appear in the
+        # actual results or only a PASS expectation appears in the actual
+        # results.
         builders_checked = []
         for config in test_expectation_line.matching_configurations:
             builder_name = self._host.builders.builder_name_for_specifiers(config.version, config.build_type)
@@ -136,7 +137,8 @@ class RemoveFlakesOMatic(object):
 
             results_for_single_test = results_by_path[test_expectation_line.path]
 
-            if self._expectations_that_were_met(test_expectation_line, results_for_single_test) != set(['PASS']):
+            expectations_met = self._expectations_that_were_met(test_expectation_line, results_for_single_test)
+            if expectations_met != set(['PASS']) and expectations_met != set([]):
                 return False
 
         if builders_checked:

third_party/blink/tools/blinkpy/web_tests/update_flaky_expectations_unittest.py

@@ -394,6 +394,38 @@ class UpdateTestExpectationsTest(LoggingTestCase):
             """# Keep since it's all failures.
             Bug(test) test/a.html [ Failure Pass ]"""))
 
+    def test_remove_none_met(self):
+        """Tests that expectations with no matching result are removed.
+
+        Expectations that are failing in a different way than specified should
+        be removed, even if there is no passing result.
+        """
+        test_expectations_before = (
+            """# Remove these two since CRASH and TIMEOUT aren't considered
+            # Failure.
+            Bug(test) test/a.html [ Failure Pass ]
+            Bug(test) test/b.html [ Failure Pass ]""")
+
+        self._define_builders({
+            'WebKit Linux Trusty': {
+                'port_name': 'linux-trusty',
+                'specifiers': ['Trusty', 'Release']
+            },
+        })
+        self._port.all_build_types = ('release',)
+        self._port.all_systems = (('trusty', 'x86_64'),)
+
+        self._parse_expectations(test_expectations_before)
+        self._expectation_factory.all_results_by_builder = {
+            'WebKit Linux Trusty': {
+                'test/a.html': ['CRASH'],
+                'test/b.html': ['TIMEOUT'],
+            }
+        }
+        updated_expectations = (
+            self._flake_remover.get_updated_test_expectations())
+        self._assert_expectations_match(updated_expectations, (''))
+
     def test_empty_test_expectations(self):
         """Running on an empty TestExpectations file outputs an empty file."""
         test_expectations_before = ''