DOM-object leak detection at run_webkit_tests.py

Now we're implementing the DOM-object leak detector. When the number of
DOM objects is increased after a test, we can assume that the test
causes a leak. This feature is available at content_shell with the flag
--enable-leak-detection.

This patch enables this leak detection feature at run_webkit_test.py
with the flag --enable-leak-detection.

content_shell is assumed to report a detected leak with the following
format:

  #LEAK - renderer pid PID (DETAIL_IN_JSON_FORMAT)

and run_webkit_test.py will treat this as a kind of error by this patch.

See also:
https://docs.google.com/a/chromium.org/document/d/1sFAsZxeISKnbGdXoLZlB2tDZ8pvO102ePFQx6TX4X14/edit

BUG=332630
TEST=test-webkitpy

Review URL: https://codereview.chromium.org/148153009

git-svn-id: svn://svn.chromium.org/blink/trunk@170255 bbb929c8-8fbe-4397-9dbb-9b2b20218538

third_party/WebKit/LayoutTests/fast/harness/results.html

@@ -206,6 +206,7 @@ function globalState()
     if (!g_state) {
         g_state = {
             crashTests: [],
+            leakTests: [],
             flakyPassTests: [],
             hasHttpTests: false,
             hasImageFailures: false,
@@ -491,6 +492,11 @@ function processGlobalStateFor(testObject)
         return;
     }
 
+    if (actual == 'LEAK') {
+        globalState().leakTests.push(testObject);
+        return;
+    }
+
     if (actual == 'TIMEOUT') {
         globalState().timeoutTests.push(testObject);
         return;
@@ -681,7 +687,9 @@ function testList(tests, header, tableId)
         else if (tableId == 'crash-tests-table') {
             html += resultLink(stripExtension(test), '-crash-log.txt', 'crash log');
             html += resultLink(stripExtension(test), '-sample.txt', 'sample');
-        } else if (tableId == 'timeout-tests-table') {
+        } else if (tableId == 'leak-tests-table')
+            html += resultLink(stripExtension(test), '-leak-log.txt', 'leak log');
+        else if (tableId == 'timeout-tests-table') {
             // FIXME: only include timeout actual/diff results here if we actually spit out results for timeout tests.
             html += textResultLinks(stripExtension(test));
         }
@@ -1330,6 +1338,9 @@ function generatePage()
     if (globalState().crashTests.length)
         html += testList(globalState().crashTests, 'Tests that crashed', 'crash-tests-table');
 
+    if (globalState().leakTests.length)
+        html += testList(globalState().leakTests, 'Tests that leaked', 'leak-tests-table');
+
     html += failingTestsTable(globalState().failingTests,
         'Tests that failed text/pixel/audio diff', 'results-table');
 

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/single_test_runner.py

@@ -220,6 +220,10 @@ class SingleTestRunner(object):
                 _log.debug("%s %s crashed, (stderr lines):" % (self._worker_name, testname))
             else:
                 _log.debug("%s %s crashed, (no stderr)" % (self._worker_name, testname))
+        elif driver_output.leak:
+            failures.append(test_failures.FailureLeak(bool(reference_filename),
+                                                      driver_output.leak_log))
+            _log.debug("%s %s leaked" % (self._worker_name, testname))
         elif driver_output.error:
             _log.debug("%s %s output stderr lines:" % (self._worker_name, testname))
         for line in driver_output.error.splitlines():

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/controllers/test_result_writer.py

@@ -65,6 +65,8 @@ def write_test_result(filesystem, port, results_directory, test_name, driver_out
         elif isinstance(failure, test_failures.FailureCrash):
             crashed_driver_output = expected_driver_output if failure.is_reftest else driver_output
             writer.write_crash_log(crashed_driver_output.crash_log)
+        elif isinstance(failure, test_failures.FailureLeak):
+            writer.write_leak_log(driver_output.leak_log)
         elif isinstance(failure, test_failures.FailureReftestMismatch):
             writer.write_image_files(driver_output.image, expected_driver_output.image)
             # FIXME: This work should be done earlier in the pipeline (e.g., when we compare images for non-ref tests).
@@ -100,6 +102,7 @@ class TestResultWriter(object):
     FILENAME_SUFFIX_STDERR = "-stderr"
     FILENAME_SUFFIX_CRASH_LOG = "-crash-log"
     FILENAME_SUFFIX_SAMPLE = "-sample"
+    FILENAME_SUFFIX_LEAK_LOG = "-leak-log"
     FILENAME_SUFFIX_WDIFF = "-wdiff.html"
     FILENAME_SUFFIX_PRETTY_PATCH = "-pretty-diff.html"
     FILENAME_SUFFIX_IMAGE_DIFF = "-diff.png"
@@ -169,6 +172,10 @@ class TestResultWriter(object):
         filename = self.output_filename(self.FILENAME_SUFFIX_CRASH_LOG + ".txt")
         self._write_file(filename, crash_log.encode('utf8', 'replace'))
 
+    def write_leak_log(self, leak_log):
+        filename = self.output_filename(self.FILENAME_SUFFIX_LEAK_LOG + ".txt")
+        self._write_file(filename, leak_log)
+
     def copy_sample_file(self, sample_file):
         filename = self.output_filename(self.FILENAME_SUFFIX_SAMPLE + ".txt")
         self._filesystem.copyfile(sample_file, filename)

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_expectations.py

@@ -42,8 +42,8 @@ _log = logging.getLogger(__name__)
 #
 # FIXME: range() starts with 0 which makes if expectation checks harder
 # as PASS is 0.
-(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, SKIP, WONTFIX,
- SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(18)
+(PASS, FAIL, TEXT, IMAGE, IMAGE_PLUS_TEXT, AUDIO, TIMEOUT, CRASH, LEAK, SKIP, WONTFIX,
+ SLOW, REBASELINE, NEEDS_REBASELINE, NEEDS_MANUAL_REBASELINE, MISSING, FLAKY, NOW, NONE) = range(19)
 
 # FIXME: Perhas these two routines should be part of the Port instead?
 BASELINE_SUFFIX_LIST = ('png', 'wav', 'txt')
@@ -222,6 +222,7 @@ class TestExpectationParser(object):
     # FIXME: Update the original specifiers list and remove this once the old syntax is gone.
     _expectation_tokens = {
         'Crash': 'CRASH',
+        'Leak': 'LEAK',
         'Failure': 'FAIL',
         'ImageOnlyFailure': 'IMAGE',
         MISSING_KEYWORD: 'MISSING',
@@ -806,6 +807,7 @@ class TestExpectations(object):
                     'text': TEXT,
                     'timeout': TIMEOUT,
                     'crash': CRASH,
+                    'leak': LEAK,
                     'missing': MISSING,
                     TestExpectationParser.SKIP_MODIFIER: SKIP,
                     TestExpectationParser.NEEDS_REBASELINE_MODIFIER: NEEDS_REBASELINE,
@@ -826,6 +828,7 @@ class TestExpectations(object):
                                 IMAGE_PLUS_TEXT: 'image and text failures',
                                 AUDIO: 'audio failures',
                                 CRASH: 'crashes',
+                                LEAK: 'leaks',
                                 TIMEOUT: 'timeouts',
                                 MISSING: 'missing results'}
 

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_failures.py

@@ -51,6 +51,8 @@ def determine_result_type(failure_list):
     failure_types = [type(f) for f in failure_list]
     if FailureCrash in failure_types:
         return test_expectations.CRASH
+    elif FailureLeak in failure_types:
+        return test_expectations.LEAK
     elif FailureTimeout in failure_types:
         return test_expectations.TIMEOUT
     elif FailureEarlyExit in failure_types:
@@ -137,6 +139,16 @@ class FailureCrash(TestFailure):
         return True
 
 
+class FailureLeak(TestFailure):
+    def __init__(self, is_reftest=False, log=''):
+        super(FailureLeak, self).__init__()
+        self.is_reftest = is_reftest
+        self.log = log
+
+    def message(self):
+        return "leak detected: %s" % (self.log)
+
+
 class FailureMissingResult(TestFailure):
     def message(self):
         return "-expected.txt was missing"

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/models/test_run_results_unittest.py

@@ -126,13 +126,13 @@ class SummarizedResultsTest(unittest.TestCase):
 
     def test_num_failures_by_type(self):
         summary = summarized_results(self.port, expected=False, passing=False, flaky=False)
-        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 2, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1})
+        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 0, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 2, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 1})
 
         summary = summarized_results(self.port, expected=True, passing=False, flaky=False)
-        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0})
+        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 1, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 1, 'REBASELINE': 0, 'SKIP': 0, 'SLOW': 0, 'TIMEOUT': 1, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 1, 'WONTFIX': 0})
 
         summary = summarized_results(self.port, expected=False, passing=True, flaky=False)
-        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 4, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0})
+        self.assertEquals(summary['num_failures_by_type'], {'CRASH': 0, 'MISSING': 0, 'TEXT': 0, 'IMAGE': 0, 'NEEDSREBASELINE': 0, 'NEEDSMANUALREBASELINE': 0, 'PASS': 4, 'REBASELINE': 0, 'SKIP': 1, 'SLOW': 0, 'TIMEOUT': 0, 'IMAGE+TEXT': 0, 'LEAK': 0, 'FAIL': 0, 'AUDIO': 0, 'WONTFIX': 0})
 
     def test_svn_revision(self):
         self.port._options.builder_name = 'dummy builder'

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/base.py

@@ -1512,6 +1512,10 @@ class Port(object):
             '\n'.join(('STDOUT: ' + l) for l in stdout_lines),
             '\n'.join(('STDERR: ' + l) for l in stderr_lines)))
 
+    def _get_leak_log(self, name, pid, stdout, stderr, newer_than):
+        match = re.match('#LEAK - (\S+) pid (\d+) (.+)\n', stderr)
+        return (stderr, match.group(3))
+
     def look_for_new_crash_logs(self, crashed_processes, start_time):
         pass
 

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/driver.py

@@ -60,7 +60,7 @@ class DriverOutput(object):
 
     def __init__(self, text, image, image_hash, audio, crash=False,
             test_time=0, measurements=None, timeout=False, error='', crashed_process_name='??',
-            crashed_pid=None, crash_log=None, pid=None):
+            crashed_pid=None, crash_log=None, leak=False, leak_log=None, pid=None):
         # FIXME: Args could be renamed to better clarify what they do.
         self.text = text
         self.image = image  # May be empty-string if the test crashes.
@@ -71,6 +71,8 @@ class DriverOutput(object):
         self.crashed_process_name = crashed_process_name
         self.crashed_pid = crashed_pid
         self.crash_log = crash_log
+        self.leak = leak
+        self.leak_log = leak_log
         self.test_time = test_time
         self.measurements = measurements
         self.timeout = timeout
@@ -102,17 +104,22 @@ class Driver(object):
         self._no_timeout = no_timeout
 
         self._driver_tempdir = None
-        # WebKitTestRunner can report back subprocess crashes by printing
+        # content_shell can report back subprocess crashes by printing
         # "#CRASHED - PROCESSNAME".  Since those can happen at any time
         # and ServerProcess won't be aware of them (since the actual tool
         # didn't crash, just a subprocess) we record the crashed subprocess name here.
         self._crashed_process_name = None
         self._crashed_pid = None
 
-        # WebKitTestRunner can report back subprocesses that became unresponsive
+        # content_shell can report back subprocesses that became unresponsive
         # This could mean they crashed.
         self._subprocess_was_unresponsive = False
 
+        # content_shell can report back subprocess DOM-object leaks by printing
+        # "#LEAK". This leak detection is enabled only when the flag
+        # --enable-leak-detection is passed to content_shell.
+        self._leaked = False
+
         # stderr reading is scoped on a per-test (not per-block) basis, so we store the accumulated
         # stderr output, as well as if we've seen #EOF on this driver instance.
         # FIXME: We should probably remove _read_first_block and _read_optional_image_block and
@@ -158,8 +165,9 @@ class Driver(object):
         crashed = self.has_crashed()
         timed_out = self._server_process.timed_out
         pid = self._server_process.pid()
+        leaked = self._leaked
 
-        if stop_when_done or crashed or timed_out:
+        if stop_when_done or crashed or timed_out or leaked:
             # We call stop() even if we crashed or timed out in order to get any remaining stdout/stderr output.
             # In the timeout case, we kill the hung process as well.
             out, err = self._server_process.stop(self._port.driver_stop_timeout() if stop_when_done else 0.0)
@@ -170,6 +178,7 @@ class Driver(object):
             self._server_process = None
 
         crash_log = None
+        leak_log = None
         if crashed:
             self.error_from_test, crash_log = self._get_crash_log(text, self.error_from_test, newer_than=start_time)
 
@@ -184,16 +193,23 @@ class Driver(object):
                 # Print stdout and stderr to the placeholder crash log; we want as much context as possible.
                 if self.error_from_test:
                     crash_log += '\nstdout:\n%s\nstderr:\n%s\n' % (text, self.error_from_test)
+        elif leaked:
+            self.error_from_test, leak_log = self._get_leak_log(text, self.error_from_test, newer_than=start_time)
 
         return DriverOutput(text, image, actual_image_hash, audio,
             crash=crashed, test_time=time.time() - test_begin_time, measurements=self._measurements,
             timeout=timed_out, error=self.error_from_test,
             crashed_process_name=self._crashed_process_name,
-            crashed_pid=self._crashed_pid, crash_log=crash_log, pid=pid)
+            crashed_pid=self._crashed_pid, crash_log=crash_log,
+            leak=leaked, leak_log=leak_log,
+            pid=pid)
 
     def _get_crash_log(self, stdout, stderr, newer_than):
         return self._port._get_crash_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than)
 
+    def _get_leak_log(self, stdout, stderr, newer_than):
+        return self._port._get_leak_log(self._crashed_process_name, self._crashed_pid, stdout, stderr, newer_than)
+
     # FIXME: Seems this could just be inlined into callers.
     @classmethod
     def _command_wrapper(cls, wrapper_option):
@@ -268,6 +284,7 @@ class Driver(object):
         environment = self._setup_environ_for_driver(environment)
         self._crashed_process_name = None
         self._crashed_pid = None
+        self._leaked = False
         cmd_line = self.cmd_line(pixel_tests, per_test_args)
         self._server_process = self._port._server_process_constructor(self._port, server_name, cmd_line, environment, logging=self._port.get_option("driver_logging"))
         self._server_process.start()
@@ -320,6 +337,8 @@ class Driver(object):
             cmd.append('--no-timeout')
         cmd.extend(self._port.get_option('additional_drt_flag', []))
         cmd.extend(self._port.additional_drt_flag())
+        if self._port.get_option('enable_leak_detection'):
+            cmd.append('--enable-leak-detection')
         cmd.extend(per_test_args)
         cmd.append('-')
         return cmd
@@ -348,6 +367,11 @@ class Driver(object):
             return True
         return self.has_crashed()
 
+    def _check_for_leak(self, error_line):
+        if error_line.startswith("#LEAK - "):
+            self._leaked = True
+        return self._leaked
+
     def _command_from_driver_input(self, driver_input):
         # FIXME: performance tests pass in full URLs instead of test names.
         if driver_input.test_name.startswith('http://') or driver_input.test_name.startswith('https://')  or driver_input.test_name == ('about:blank'):
@@ -461,6 +485,7 @@ class Driver(object):
             if err_line:
                 if self._check_for_driver_crash(err_line):
                     break
+                self._check_for_leak(err_line)
                 self.error_from_test += err_line
 
         block.decode_content()

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/port/driver_unittest.py

@@ -150,11 +150,12 @@ class DriverTest(unittest.TestCase):
             def stop(self, timeout=0.0):
                 pass
 
-        def assert_crash(driver, error_line, crashed, name, pid, unresponsive=False):
+        def assert_crash(driver, error_line, crashed, name, pid, unresponsive=False, leaked=False):
             self.assertEqual(driver._check_for_driver_crash(error_line), crashed)
             self.assertEqual(driver._crashed_process_name, name)
             self.assertEqual(driver._crashed_pid, pid)
             self.assertEqual(driver._subprocess_was_unresponsive, unresponsive)
+            self.assertEqual(driver._check_for_leak(error_line), leaked)
             driver.stop()
 
         driver._server_process = FakeServerProcess(False)
@@ -164,36 +165,49 @@ class DriverTest(unittest.TestCase):
         driver._crashed_pid = None
         driver._server_process = FakeServerProcess(False)
         driver._subprocess_was_unresponsive = False
+        driver._leaked = False
         assert_crash(driver, '#CRASHED\n', True, 'FakeServerProcess', 1234)
 
         driver._crashed_process_name = None
         driver._crashed_pid = None
         driver._server_process = FakeServerProcess(False)
         driver._subprocess_was_unresponsive = False
+        driver._leaked = False
         assert_crash(driver, '#CRASHED - WebProcess\n', True, 'WebProcess', None)
 
         driver._crashed_process_name = None
         driver._crashed_pid = None
         driver._server_process = FakeServerProcess(False)
         driver._subprocess_was_unresponsive = False
+        driver._leaked = False
         assert_crash(driver, '#CRASHED - WebProcess (pid 8675)\n', True, 'WebProcess', 8675)
 
         driver._crashed_process_name = None
         driver._crashed_pid = None
         driver._server_process = FakeServerProcess(False)
         driver._subprocess_was_unresponsive = False
+        driver._leaked = False
         assert_crash(driver, '#PROCESS UNRESPONSIVE - WebProcess (pid 8675)\n', True, 'WebProcess', 8675, True)
 
         driver._crashed_process_name = None
         driver._crashed_pid = None
         driver._server_process = FakeServerProcess(False)
         driver._subprocess_was_unresponsive = False
+        driver._leaked = False
         assert_crash(driver, '#CRASHED - renderer (pid 8675)\n', True, 'renderer', 8675)
 
+        driver._crashed_process_name = None
+        driver._crashed_pid = None
+        driver._server_process = FakeServerProcess(False)
+        driver._subprocess_was_unresponsive = False
+        driver._leaked = False
+        assert_crash(driver, '#LEAK - renderer pid 8675 ({"numberOfLiveDocuments":[2,3]})\n', False, None, None, False, True)
+
         driver._crashed_process_name = None
         driver._crashed_pid = None
         driver._server_process = FakeServerProcess(True)
         driver._subprocess_was_unresponsive = False
+        driver._leaked = False
         assert_crash(driver, '', True, 'FakeServerProcess', 1234)
 
     def test_creating_a_port_does_not_write_to_the_filesystem(self):

third_party/WebKit/Tools/Scripts/webkitpy/layout_tests/run_webkit_tests.py

@@ -257,6 +257,8 @@ def parse_args(args):
             help="Use Apache instead of LigHTTPd (default is port-specific)."),
         optparse.make_option("--no-use-apache", action="store_false", dest="use_apache",
             help="Use LigHTTPd instead of Apache (default is port-specific)."),
+        optparse.make_option("--enable-leak-detection", action="store_true",
+            help="Enable the leak detection of DOM objects."),
     ]))
 
     option_group_definitions.append(("Miscellaneous Options", [