2011-03-15 Hayato Ito <hayato@chromium.org>

        Reviewed by Shinichiro Hamaji.

        [NRWT] Add support for reftests to new-run-webkit-tests.

        https://bugs.webkit.org/show_bug.cgi?id=55457

        * Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
        * Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
        * Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
        * Scripts/webkitpy/layout_tests/port/base.py:
        * Scripts/webkitpy/layout_tests/port/dryrun.py:
        * Scripts/webkitpy/layout_tests/port/test.py:
        * Scripts/webkitpy/layout_tests/port/test_files.py:
        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:

git-svn-id: svn://svn.chromium.org/blink/trunk@81127 bbb929c8-8fbe-4397-9dbb-9b2b20218538

third_party/WebKit/Tools/ChangeLog

+2011-03-15  Hayato Ito  <hayato@chromium.org>
+
+        Reviewed by Shinichiro Hamaji.
+
+        [NRWT] Add support for reftests to new-run-webkit-tests.
+
+        https://bugs.webkit.org/show_bug.cgi?id=55457
+
+        * Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py:
+        * Scripts/webkitpy/layout_tests/layout_package/test_failures.py:
+        * Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py:
+        * Scripts/webkitpy/layout_tests/port/base.py:
+        * Scripts/webkitpy/layout_tests/port/dryrun.py:
+        * Scripts/webkitpy/layout_tests/port/test.py:
+        * Scripts/webkitpy/layout_tests/port/test_files.py:
+        * Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py:
+
 2011-03-15  Alejandro G. Castro  <alex@igalia.com>
 
         Reviewed by Martin Robinson.

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/single_test_runner.py

@@ -44,14 +44,6 @@ def run_single_test(port, options, test_input, driver, worker_name):
     return runner.run()
 
 
-class ExpectedDriverOutput:
-    """Groups information about an expected driver output."""
-    def __init__(self, text, image, image_hash):
-        self.text = text
-        self.image = image
-        self.image_hash = image_hash
-
-
 class SingleTestRunner:
 
     def __init__(self, options, port, driver, test_input, worker_name):
@@ -63,10 +55,43 @@ class SingleTestRunner:
         self._worker_name = worker_name
         self._testname = port.relative_test_filename(test_input.filename)
 
+        self._is_reftest = False
+        self._is_mismatch_reftest = False
+        self._reference_filename = None
+
+        fs = port._filesystem
+        reftest_expected_filename = port.reftest_expected_filename(self._filename)
+        if fs.exists(reftest_expected_filename):
+            self._is_reftest = True
+            self._reference_filename = reftest_expected_filename
+
+        reftest_expected_mismatch_filename = port.reftest_expected_mismatch_filename(self._filename)
+        if fs.exists(reftest_expected_mismatch_filename):
+            if self._is_reftest:
+                _log.error('It is not allowed that one test file has both'
+                           ' expected.html file and expected-mismatch.html file'
+                           ' at the same time. Please remove either %s or %s.',
+                           reftest_expected_filename, reftest_expected_mismatch_filename)
+            else:
+                self._is_reftest = True
+                self._is_mismatch_reftest = True
+                self._reference_filename = reftest_expected_mismatch_filename
+
+        if self._is_reftest:
+            # Detect and report a test which has a wrong combination of expectation files.
+            # For example, if 'foo.html' has two expectation files, 'foo-expected.html' and
+            # 'foo-expected.txt', we should warn users. One test file must be used exclusively
+            # in either layout tests or reftests, but not in both.
+            for suffix in ['.txt', '.checksum', '.png']:
+                expected_filename = self._port.expected_filename(self._filename, suffix)
+                if fs.exists(expected_filename):
+                    _log.error('The reftest (%s) can not have an expectation file (%s).'
+                               ' Please remove that file.', self._testname, expected_filename)
+
     def _expected_driver_output(self):
-        return ExpectedDriverOutput(self._port.expected_text(self._filename),
-                                    self._port.expected_image(self._filename),
-                                    self._port.expected_checksum(self._filename))
+        return base.DriverOutput(self._port.expected_text(self._filename),
+                                 self._port.expected_image(self._filename),
+                                 self._port.expected_checksum(self._filename))
 
     def _should_fetch_expected_checksum(self):
         return (self._options.pixel_tests and
@@ -84,7 +109,13 @@ class SingleTestRunner:
 
     def run(self):
         if self._options.new_baseline or self._options.reset_results:
-            return self._run_rebaseline()
+            if self._is_reftest:
+                # Returns a dummy TestResult. We don't have to rebase for reftests.
+                return TestResult(self._filename)
+            else:
+                return self._run_rebaseline()
+        if self._is_reftest:
+            return self._run_reftest()
         return self._run_compare_test()
 
     def _run_compare_test(self):
@@ -98,6 +129,8 @@ class SingleTestRunner:
     def _run_rebaseline(self):
         driver_output = self._driver.run_test(self._driver_input())
         failures = self._handle_error(driver_output)
+        test_result_writer.write_test_result(self._port, self._options.results_directory, self._filename,
+                                             driver_output, None, failures)
         # FIXME: It the test crashed or timed out, it might be bettter to avoid
         # to write new baselines.
         self._save_baselines(driver_output)
@@ -145,22 +178,31 @@ class SingleTestRunner:
 
         port.update_baseline(output_path, data)
 
-    def _handle_error(self, driver_output):
+    def _handle_error(self, driver_output, reference_filename=None):
+        """Returns test failures if some unusual errors happen in driver's run.
+
+        Args:
+          driver_output: The output from the driver.
+          reference_filename: The full path to the reference file which produced the driver_output.
+              This arg is optional and should be used only in reftests until we have a better way to know
+              which html file is used for producing the driver_output.
+        """
         failures = []
         fs = self._port._filesystem
         if driver_output.timeout:
-            failures.append(test_failures.FailureTimeout())
+            failures.append(test_failures.FailureTimeout(reference_filename))
+
+        if reference_filename:
+            testname = self._port.relative_test_filename(reference_filename)
+        else:
+            testname = self._testname
+
         if driver_output.crash:
-            failures.append(test_failures.FailureCrash())
-            _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, self._testname,
+            failures.append(test_failures.FailureCrash(reference_filename))
+            _log.debug("%s Stacktrace for %s:\n%s" % (self._worker_name, testname,
                                                       driver_output.error))
-            # FIXME: Use test_result_writer module.
-            stack_filename = fs.join(self._options.results_directory, self._testname)
-            stack_filename = fs.splitext(stack_filename)[0] + "-stack.txt"
-            fs.maybe_make_directory(fs.dirname(stack_filename))
-            fs.write_text_file(stack_filename, driver_output.error)
         elif driver_output.error:
-            _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, self._testname,
+            _log.debug("%s %s output stderr lines:\n%s" % (self._worker_name, testname,
                                                            driver_output.error))
         return failures
 
@@ -210,3 +252,31 @@ class SingleTestRunner:
         elif driver_output.image_hash != expected_driver_outputs.image_hash:
             failures.append(test_failures.FailureImageHashMismatch())
         return failures
+
+    def _run_reftest(self):
+        driver_output1 = self._driver.run_test(self._driver_input())
+        driver_output2 = self._driver.run_test(
+            base.DriverInput(self._reference_filename, self._timeout, driver_output1.image_hash))
+        test_result = self._compare_output_with_reference(driver_output1, driver_output2)
+
+        test_result_writer.write_test_result(self._port, self._options.results_directory, self._filename,
+                                             driver_output1, driver_output2, test_result.failures)
+        return test_result
+
+    def _compare_output_with_reference(self, driver_output1, driver_output2):
+        total_test_time = driver_output1.test_time + driver_output2.test_time
+        failures = []
+        failures.extend(self._handle_error(driver_output1))
+        if failures:
+            # Don't continue any more if we already have crash or timeout.
+            return TestResult(self._filename, failures, total_test_time)
+        failures.extend(self._handle_error(driver_output2, reference_filename=self._reference_filename))
+        if failures:
+            return TestResult(self._filename, failures, total_test_time)
+
+        if self._is_mismatch_reftest:
+            if driver_output1.image_hash == driver_output2.image_hash:
+                failures.append(test_failures.FailureReftestMismatchDidNotOccur())
+        elif driver_output1.image_hash != driver_output2.image_hash:
+            failures.append(test_failures.FailureReftestMismatch())
+        return TestResult(self._filename, failures, total_test_time)

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/test_failures.py

@@ -60,11 +60,13 @@ def determine_result_type(failure_list):
         is_text_failure = FailureTextMismatch in failure_types
         is_image_failure = (FailureImageHashIncorrect in failure_types or
                             FailureImageHashMismatch in failure_types)
+        is_reftest_failure = (FailureReftestMismatch in failure_types or
+                              FailureReftestMismatchDidNotOccur in failure_types)
         if is_text_failure and is_image_failure:
             return test_expectations.IMAGE_PLUS_TEXT
         elif is_text_failure:
             return test_expectations.TEXT
-        elif is_image_failure:
+        elif is_image_failure or is_reftest_failure:
             return test_expectations.IMAGE
         else:
             raise ValueError("unclassifiable set of failures: "
@@ -177,11 +179,17 @@ class FailureTimeout(TestFailure):
     """Test timed out.  We also want to restart DumpRenderTree if this
     happens."""
 
+    def __init__(self, reference_filename=None):
+        self.reference_filename = reference_filename
+
     @staticmethod
     def message():
         return "Test timed out"
 
     def result_html_output(self, filename):
+        if self.reference_filename:
+            return "<strong>%s</strong> (occured in <a href=%s>expected html</a>)" % (
+                self.message(), self.reference_filename)
         return "<strong>%s</strong>" % self.message()
 
     def should_kill_dump_render_tree(self):
@@ -191,6 +199,9 @@ class FailureTimeout(TestFailure):
 class FailureCrash(TestFailure):
     """DumpRenderTree crashed."""
 
+    def __init__(self, reference_filename=None):
+        self.reference_filename = reference_filename
+
     @staticmethod
     def message():
         return "DumpRenderTree crashed"
@@ -198,8 +209,11 @@ class FailureCrash(TestFailure):
     def result_html_output(self, filename):
         # FIXME: create a link to the minidump file
         stack = self.relative_output_filename(filename, "-stack.txt")
-        return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(),
-                                                             stack)
+        if self.reference_filename:
+            return "<strong>%s</strong> <a href=%s>stack</a> (occured in <a href=%s>expected html</a>)" % (
+                self.message(), stack, self.reference_filename)
+        else:
+            return "<strong>%s</strong> <a href=%s>stack</a>" % (self.message(), stack)
 
     def should_kill_dump_render_tree(self):
         return True
@@ -279,9 +293,48 @@ class FailureImageHashIncorrect(ComparisonTestFailure):
     def result_html_output(self, filename):
         return "<strong>%s</strong>" % self.message()
 
+
+class FailureReftestMismatch(ComparisonTestFailure):
+    """The result didn't match the reference rendering."""
+
+    OUT_FILENAMES = ("-expected.html", "-expected.png", "-actual.png",
+                     "-diff.png",)
+
+    @staticmethod
+    def message():
+        return "Mismatch with reference"
+
+    def output_links(self, filename, out_names):
+        links = ['']
+        uris = [self.relative_output_filename(filename, output_filename)
+                for output_filename in out_names]
+        for text, uri in zip(['-expected.html', 'expected', 'actual', 'diff'], uris):
+            links.append("<a href='%s'>%s</a>" % (uri, text))
+        return ' '.join(links)
+
+
+class FailureReftestMismatchDidNotOccur(ComparisonTestFailure):
+    """Unexpected match between the result and the reference rendering."""
+
+    OUT_FILENAMES = ("-expected-mismatch.html", "-actual.png",)
+
+    @staticmethod
+    def message():
+        return "Mismatch with the reference did not occur"
+
+    def output_links(self, filename, out_names):
+        links = ['']
+        uris = [self.relative_output_filename(filename, output_filename)
+                for output_filename in out_names]
+        for text, uri in zip(['-expected-mismatch.html', 'image'], uris):
+            links.append("<a href='%s'>%s</a>" % (uri, text))
+        return ' '.join(links)
+
+
 # Convenient collection of all failure classes for anything that might
 # need to enumerate over them all.
 ALL_FAILURE_CLASSES = (FailureTimeout, FailureCrash, FailureMissingResult,
                        FailureTextMismatch, FailureMissingImageHash,
                        FailureMissingImage, FailureImageHashMismatch,
-                       FailureImageHashIncorrect)
+                       FailureImageHashIncorrect, FailureReftestMismatch,
+                       FailureReftestMismatchDidNotOccur)

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/layout_package/test_result_writer.py

@@ -64,7 +64,17 @@ def write_test_result(port, root_output_dir, filename, driver_output,
                 checksums_mismatch_but_images_are_same = True
                 imagehash_mismatch_failure = failure
         elif isinstance(failure, test_failures.FailureCrash):
-            writer.write_crash_report(driver_output.error)
+            if failure.reference_filename:
+                writer.write_crash_report(expected_driver_output.error)
+            else:
+                writer.write_crash_report(driver_output.error)
+        elif isinstance(failure, test_failures.FailureReftestMismatch):
+            writer.write_image_files(driver_output.image, expected_driver_output.image)
+            writer.create_image_diff_and_write_result(driver_output.image, expected_driver_output.image)
+            writer.copy_file(port.reftest_expected_filename(filename), '-expected.html')
+        elif isinstance(failure, test_failures.FailureReftestMismatchDidNotOccur):
+            writer.write_image_files(driver_output.image, expected_image=None)
+            writer.copy_file(port.reftest_expected_mismatch_filename(filename), '-expected-mismatch.html')
         else:
             assert isinstance(failure, (test_failures.FailureTimeout,))
 
@@ -135,9 +145,9 @@ class TestResultWriter(object):
         expected_filename = self.output_filename(self.FILENAME_SUFFIX_EXPECTED + file_type)
 
         fs = self._port._filesystem
-        if output:
+        if output is not None:
             fs.write_binary_file(actual_filename, output)
-        if expected:
+        if expected is not None:
             fs.write_binary_file(expected_filename, expected)
 
     def write_crash_report(self, error):
@@ -193,3 +203,10 @@ class TestResultWriter(object):
         # To do so, we have to change port.diff_image() as well.
         diff_filename = self.output_filename(self.FILENAME_SUFFIX_IMAGE_DIFF)
         return self._port.diff_image(actual_image, expected_image, diff_filename)
+
+    def copy_file(self, src_filepath, dst_extension):
+        fs = self._port._filesystem
+        assert fs.exists(src_filepath), 'src_filepath: %s' % src_filepath
+        dst_filename = self.output_filename(dst_extension)
+        self._make_output_directory()
+        fs.copyfile(src_filepath, dst_filename)

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

@@ -347,6 +347,14 @@ class Port(object):
         text = self._filesystem.read_binary_file(path)
         return text.replace("\r\n", "\n")
 
+    def reftest_expected_filename(self, filename):
+        """Return the filename of reference we expect the test matches."""
+        return self.expected_filename(filename, '.html')
+
+    def reftest_expected_mismatch_filename(self, filename):
+        """Return the filename of reference we don't expect the test matches."""
+        return self.expected_filename(filename, '-mismatch.html')
+
     def filename_to_uri(self, filename):
         """Convert a test file (which is an absolute path) to a URI."""
         LAYOUTTEST_HTTP_DIR = "http/tests/"

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/dryrun.py

@@ -118,12 +118,26 @@ class DryrunDriver(base.Driver):
 
     def run_test(self, driver_input):
         start_time = time.time()
-        text_output = self._port.expected_text(driver_input.filename)
-
-        if driver_input.image_hash is not None:
+        fs = self._port._filesystem
+        if fs.exists(self._port.reftest_expected_filename(driver_input.filename)) or \
+            fs.exists(self._port.reftest_expected_mismatch_filename(driver_input.filename)):
+            text_output = 'test-text'
+            image = 'test-image'
+            hash = 'test-checksum'
+        elif driver_input.filename.endswith('-expected.html'):
+            text_output = 'test-text'
+            image = 'test-image'
+            hash = 'test-checksum'
+        elif driver_input.filename.endswith('-expected-mismatch.html'):
+            text_output = 'test-text-mismatch'
+            image = 'test-image-mismatch'
+            hash = 'test-checksum-mismatch'
+        elif driver_input.image_hash is not None:
+            text_output = self._port.expected_text(driver_input.filename)
             image = self._port.expected_image(driver_input.filename)
             hash = self._port.expected_checksum(driver_input.filename)
         else:
+            text_output = self._port.expected_text(driver_input.filename)
             image = None
             hash = None
         return base.DriverOutput(text_output, image, hash, False,

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test.py

@@ -50,6 +50,7 @@ class TestInstance:
         self.keyboard = False
         self.error = ''
         self.timeout = False
+        self.is_reftest = False
 
         # The values of each field are treated as raw byte strings. They
         # will be converted to unicode strings where appropriate using
@@ -78,6 +79,13 @@ class TestList:
             test.__dict__[key] = value
         self.tests[name] = test
 
+    def add_reftest(self, name, reference_name, same_image):
+        self.add(name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
+        if same_image:
+            self.add(reference_name, actual_checksum='xxx', actual_image='XXX', is_reftest=True)
+        else:
+            self.add(reference_name, actual_checksum='yyy', actual_image='YYY', is_reftest=True)
+
     def keys(self):
         return self.tests.keys()
 
@@ -129,6 +137,16 @@ def unit_test_list():
     # helpfully filtered to "\r\r\n" by our Python/Cygwin tooling.
     tests.add('passes/text.html',
               expected_text='\nfoo\n\n', actual_text='\nfoo\r\n\r\r\n')
+
+    # For reftests.
+    tests.add_reftest('passes/reftest.html', 'passes/reftest-expected.html', same_image=True)
+    tests.add_reftest('passes/mismatch.html', 'passes/mismatch-expected-mismatch.html', same_image=False)
+    tests.add_reftest('failures/expected/reftest.html', 'failures/expected/reftest-expected.html', same_image=False)
+    tests.add_reftest('failures/expected/mismatch.html', 'failures/expected/mismatch-expected-mismatch.html', same_image=True)
+    tests.add_reftest('failures/unexpected/reftest.html', 'failures/unexpected/reftest-expected.html', same_image=False)
+    tests.add_reftest('failures/unexpected/mismatch.html', 'failures/unexpected/mismatch-expected-mismatch.html', same_image=True)
+    # FIXME: Add a reftest which crashes.
+
     tests.add('websocket/tests/passes/text.html')
     return tests
 
@@ -158,6 +176,8 @@ def unit_test_filesystem(files=None):
     # Add each test and the expected output, if any.
     for test in test_list.tests.values():
         add_file(files, test, '.html', '')
+        if test.is_reftest:
+            continue
         add_file(files, test, '-expected.txt', test.expected_text)
         add_file(files, test, '-expected.checksum', test.expected_checksum)
         add_file(files, test, '-expected.png', test.expected_image)
@@ -169,12 +189,14 @@ WONTFIX : failures/expected/crash.html = CRASH
 // This one actually passes because the checksums will match.
 WONTFIX : failures/expected/image.html = PASS
 WONTFIX : failures/expected/image_checksum.html = IMAGE
+WONTFIX : failures/expected/mismatch.html = IMAGE
 WONTFIX : failures/expected/missing_check.html = MISSING PASS
 WONTFIX : failures/expected/missing_image.html = MISSING PASS
 WONTFIX : failures/expected/missing_text.html = MISSING PASS
 WONTFIX : failures/expected/newlines_leading.html = TEXT
 WONTFIX : failures/expected/newlines_trailing.html = TEXT
 WONTFIX : failures/expected/newlines_with_excess_CR.html = TEXT
+WONTFIX : failures/expected/reftest.html = IMAGE
 WONTFIX : failures/expected/text.html = TEXT
 WONTFIX : failures/expected/timeout.html = TIMEOUT
 WONTFIX SKIP : failures/expected/hang.html = TIMEOUT

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/test_files.py

@@ -105,11 +105,10 @@ def _has_supported_extension(filesystem, filename):
     return extension in _supported_file_extensions
 
 
-def _is_reference_html_file(filename):
+def is_reference_html_file(filename):
     """Return true if the filename points to a reference HTML file."""
     if (filename.endswith('-expected.html') or
         filename.endswith('-expected-mismatch.html')):
-        _log.warn("Reftests are not supported - ignoring %s" % filename)
         return True
     return False
 
@@ -117,4 +116,4 @@ def _is_reference_html_file(filename):
 def _is_test_file(filesystem, dirname, filename):
     """Return true if the filename points to a test file."""
     return (_has_supported_extension(filesystem, filename) and
-            not _is_reference_html_file(filename))
+            not is_reference_html_file(filename))

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests_unittest.py

@@ -55,6 +55,7 @@ from webkitpy.layout_tests import port
 from webkitpy.layout_tests import run_webkit_tests
 from webkitpy.layout_tests.layout_package import dump_render_tree_thread
 from webkitpy.layout_tests.port.test import TestPort, TestDriver
+from webkitpy.layout_tests.port.test_files import is_reference_html_file
 from webkitpy.python24.versioning import compare_version
 from webkitpy.test.skip import skip_if
 
@@ -124,7 +125,8 @@ def run_and_capture(port_obj, options, parsed_args):
     return (res, buildbot_output, regular_output)
 
 
-def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False, filesystem=None):
+def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
+                  filesystem=None, include_reference_html=False):
     extra_args = extra_args or []
     if not tests_included:
         # Not including http tests since they get run out of order (that
@@ -136,6 +138,7 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
 
     test_batches = []
 
+
     class RecordingTestDriver(TestDriver):
         def __init__(self, port, worker_number):
             TestDriver.__init__(self, port, worker_number)
@@ -153,7 +156,11 @@ def get_tests_run(extra_args=None, tests_included=False, flatten_batches=False,
                 self._current_test_batch = []
                 test_batches.append(self._current_test_batch)
             test_name = self._port.relative_test_filename(test_input.filename)
-            self._current_test_batch.append(test_name)
+            # In case of reftest, one test calls the driver's run_test() twice.
+            # We should not add a reference html used by reftests to tests unless include_reference_html parameter
+            # is explicitly given.
+            if include_reference_html or not is_reference_html_file(test_input.filename):
+                self._current_test_batch.append(test_name)
             return TestDriver.run_test(self, test_input)
 
     class RecordingTestPort(TestPort):
@@ -352,7 +359,12 @@ class MainTest(unittest.TestCase):
         # Run tests including the unexpected failures.
         self._url_opened = None
         res, out, err, user = logging_run(tests_included=True)
-        self.assertEqual(res, 3)
+
+        # Update this magic number if you add an unexpected test to webkitpy.layout_tests.port.test
+        # FIXME: It's nice to have a routine in port/test.py that returns this number.
+        unexpected_tests_count = 5
+
+        self.assertEqual(res, unexpected_tests_count)
         self.assertFalse(out.empty())
         self.assertFalse(err.empty())
         self.assertEqual(user.opened_urls, ['/tmp/layout-test-results/results.html'])
@@ -516,6 +528,25 @@ class MainTest(unittest.TestCase):
         self.assertRaises(ValueError, logging_run,
                           ['--worker-model', 'unknown'])
 
+    def test_reftest_run(self):
+        tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True)
+        self.assertEquals(['passes/reftest.html'], tests_run)
+
+    def test_reftest_expected_html_should_be_ignored(self):
+        tests_run = get_tests_run(['passes/reftest-expected.html'], tests_included=True, flatten_batches=True)
+        self.assertEquals([], tests_run)
+
+    def test_reftest_driver_should_run_expected_html(self):
+        tests_run = get_tests_run(['passes/reftest.html'], tests_included=True, flatten_batches=True,
+                                  include_reference_html=True)
+        self.assertEquals(['passes/reftest.html', 'passes/reftest-expected.html'], tests_run)
+
+    def test_reftest_driver_should_run_expected_mismatch_html(self):
+        tests_run = get_tests_run(['passes/mismatch.html'], tests_included=True, flatten_batches=True,
+                                  include_reference_html=True)
+        self.assertEquals(['passes/mismatch.html', 'passes/mismatch-expected-mismatch.html'], tests_run)
+
+
 MainTest = skip_if(MainTest, sys.platform == 'cygwin' and compare_version(sys, '2.6')[0] < 0, 'new-run-webkit-tests tests hang on Cygwin Python 2.5.2')