[run_web_tests.py] Keep stderr of the last failure for repeated tests

This is useful for debugging flaky tests with --iterations=n (or
repeat-each=n) and --exit-after-n-failures=1. Previously the first stderr
was kept which might be for a pass which was confusing in results.html
where the failure results and the passing stderr were displayed.

This doesn't affect normal runs (e.g. on bots) where tests are not
repeated in one run (not including retries).

Also output stderr for timeout tests, which is also useful for debugging.

Change-Id: I25c1296e14b22763f36eb96d4028896f28a11c36
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2284215Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Xianzhu Wang <wangxianzhu@chromium.org>
Cr-Commit-Position: refs/heads/master@{#785827}

third_party/blink/tools/blinkpy/web_tests/models/test_failures.py

@@ -140,7 +140,11 @@ class AbstractTestResultType(object):
             artifacts_abspath = self.filesystem.join(
                 self.result_directory, typ_artifacts.ArtifactsSubDirectory(),
                 artifact_filename)
-            if not self.filesystem.exists(artifacts_abspath):
+            # If a test has multiple stderr results, keep that of the last
+            # failure, which is useful for debugging flaky tests with
+            # --iterations=n or --repeat-each=n.
+            if (force_overwrite or self.result != ResultType.Pass
+                    or not self.filesystem.exists(artifacts_abspath)):
                 self._write_to_artifacts(
                     typ_artifacts,
                     'stderr',
@@ -202,9 +206,6 @@ class FailureTimeout(AbstractTestResultType):
         super(FailureTimeout, self).__init__(actual_driver_output, None)
         self.is_reftest = is_reftest
 
-    def create_artifacts(self, typ_artifacts, force_overwrite=False):
-        pass
-
     def message(self):
         return 'test timed out'
 

third_party/blink/tools/blinkpy/web_tests/models/test_failures_unittest.py

@@ -27,10 +27,16 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import unittest
+from blinkpy.common.system.system_host_mock import MockSystemHost
+from blinkpy.web_tests.models.typ_types import ResultType, Artifacts
+from blinkpy.web_tests.port.base import Port
 from blinkpy.web_tests.port.driver import DriverOutput
 
-from blinkpy.web_tests.models.test_failures import (
-    ALL_FAILURE_CLASSES, FailureCrash, FailureTimeout, TestFailure)
+from blinkpy.web_tests.models.test_failures import (ALL_FAILURE_CLASSES,
+                                                    PassWithStderr,
+                                                    FailureCrash,
+                                                    FailureTimeout,
+                                                    TestFailure)
 
 
 class TestFailuresTest(unittest.TestCase):
@@ -61,31 +67,67 @@ class TestFailuresTest(unittest.TestCase):
             self.assert_loads(c)
 
     def test_equals(self):
-        self.assertEqual(
-            FailureCrash(self._actual_output, self._expected_output),
-            FailureCrash(self._actual_output, self._expected_output))
-        self.assertNotEqual(
-            FailureCrash(self._actual_output, self._expected_output),
-            FailureTimeout(self._actual_output, self._expected_output))
+        self.assertEqual(FailureCrash(self._actual_output),
+                         FailureCrash(self._actual_output))
+        self.assertNotEqual(FailureCrash(self._actual_output),
+                            FailureTimeout(self._actual_output))
         crash_set = set([
-            FailureCrash(self._actual_output, self._expected_output),
-            FailureCrash(self._actual_output, self._expected_output)
+            FailureCrash(self._actual_output),
+            FailureCrash(self._actual_output)
         ])
         self.assertEqual(len(crash_set), 1)
         # The hash happens to be the name of the class, but sets still work:
-        crash_set = set([
-            FailureCrash(self._actual_output, self._expected_output),
-            'FailureCrash'
-        ])
+        crash_set = set([FailureCrash(self._actual_output), 'FailureCrash'])
         self.assertEqual(len(crash_set), 2)
 
     def test_crashes(self):
         self.assertEqual(
-            FailureCrash(self._actual_output, self._expected_output).message(),
+            FailureCrash(self._actual_output).message(),
             'content_shell crashed')
         self.assertEqual(
             FailureCrash(
                 self._actual_output,
-                self._expected_output,
                 process_name='foo',
                 pid=1234).message(), 'foo crashed [pid=1234]')
+
+    def test_repeated_test_artifacts(self):
+        host = MockSystemHost()
+        port = Port(host, 'baseport')
+        artifacts = Artifacts('/dir', host.filesystem, repeat_tests=True)
+
+        def init_test_failure(test_failure):
+            test_failure.port = port
+            test_failure.filesystem = host.filesystem
+            test_failure.test_name = 'foo.html'
+            test_failure.result_directory = '/dir'
+
+        pass_with_stderr = PassWithStderr(
+            DriverOutput(None, None, None, None, error='pass with stderr'))
+        init_test_failure(pass_with_stderr)
+        crash = FailureCrash(
+            DriverOutput(None,
+                         None,
+                         None,
+                         None,
+                         crash=True,
+                         error='crash stderr'))
+        init_test_failure(crash)
+        timeout = FailureTimeout(
+            DriverOutput(None, None, None, None, error='timeout with stderr'))
+        init_test_failure(timeout)
+
+        pass_with_stderr.create_artifacts(artifacts)
+        self.assertEqual('pass with stderr',
+                         host.filesystem.read_text_file('/dir/foo-stderr.txt'))
+
+        crash.create_artifacts(artifacts)
+        self.assertEqual('crash stderr',
+                         host.filesystem.read_text_file('/dir/foo-stderr.txt'))
+
+        timeout.create_artifacts(artifacts)
+        self.assertEqual('timeout with stderr',
+                         host.filesystem.read_text_file('/dir/foo-stderr.txt'))
+
+        pass_with_stderr.create_artifacts(artifacts)
+        self.assertEqual('timeout with stderr',
+                         host.filesystem.read_text_file('/dir/foo-stderr.txt'))