Make /gen/ available in wptserve

/gen/ is already available in both file tests (via blink_test_runner.cc)
and http tests (via Apache 'Alias' directives). This CL makes /gen/ also
available to wptserve in Blink (both run_web_tests and
run_blink_wptserve).

Besides, this CL adds configuration options (release, debug, etc.) to
run_blink_wptserve so that users can control which out/*/gen is served.

Also fix a leaking file descriptor in filesystem.open_text_tempfile.

Bug: 821496
Change-Id: I60ae0657df470dd319d002bc1c20476d33d9c05e
Reviewed-on: https://chromium-review.googlesource.com/1140908
Commit-Queue: Robert Ma <robertma@chromium.org>
Reviewed-by: Quinten Yearsley <qyearsley@chromium.org>
Reviewed-by: Philip Jägenstedt <foolip@chromium.org>
Cr-Commit-Position: refs/heads/master@{#576210}

third_party/blink/tools/blinkpy/common/system/filesystem.py

@@ -257,7 +257,10 @@ class FileSystem(object):
 
         Returns a tuple of the file and the name.
         """
-        _, temp_name = tempfile.mkstemp(suffix)
+        temp_fd, temp_name = tempfile.mkstemp(suffix)
+        # Close the OS fd opened by mkstemp as we will reopen the file with an
+        # explict encoding.
+        os.close(temp_fd)
         f = codecs.open(temp_name, 'w', 'utf8')
         return f, temp_name
 

third_party/blink/tools/blinkpy/web_tests/servers/cli_wrapper.py

@@ -32,35 +32,43 @@ Scripts in tools/ can use this module to start servers that are normally used
 for layout tests, outside of the layout test runner.
 """
 
-import argparse
 import logging
+import optparse
 
 from blinkpy.common.host import Host
+from blinkpy.web_tests.port.factory import configuration_options
+
+
+class RawTextHelpFormatter(optparse.IndentedHelpFormatter):
+    def format_description(self, description):
+        return description
 
 
 def main(server_constructor, input_fn=None, argv=None, description=None, **kwargs):
     input_fn = input_fn or raw_input
 
-    parser = argparse.ArgumentParser(description=description, formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument('--output-dir', type=str, default=None,
+    parser = optparse.OptionParser(description=description, formatter=RawTextHelpFormatter())
+    parser.add_option('--output-dir', type=str, default=None,
                       help='output directory, for log files etc.')
-    parser.add_argument('-v', '--verbose', action='store_true',
+    parser.add_option('-v', '--verbose', action='store_true',
                       help='print more information, including port numbers')
-    args = parser.parse_args(argv)
+    for opt in configuration_options():
+        parser.add_option(opt)
+    options, _ = parser.parse_args(argv)
 
     logging.basicConfig()
     logger = logging.getLogger()
-    logger.setLevel(logging.DEBUG if args.verbose else logging.INFO)
+    logger.setLevel(logging.DEBUG if options.verbose else logging.INFO)
 
     host = Host()
-    port_obj = host.port_factory.get()
-    if not args.output_dir:
-        args.output_dir = port_obj.default_results_directory()
+    port_obj = host.port_factory.get(options=options)
+    if not options.output_dir:
+        options.output_dir = port_obj.default_results_directory()
 
     # Create the output directory if it doesn't already exist.
-    port_obj.host.filesystem.maybe_make_directory(args.output_dir)
+    port_obj.host.filesystem.maybe_make_directory(options.output_dir)
 
-    server = server_constructor(port_obj, args.output_dir, **kwargs)
+    server = server_constructor(port_obj, options.output_dir, **kwargs)
     server.start()
     try:
         _ = input_fn('Hit any key to stop the server and exit.')

third_party/blink/tools/blinkpy/web_tests/servers/wptserve.py

@@ -5,6 +5,7 @@
 """Start and stop the WPTserve servers as they're used by the layout tests."""
 
 import datetime
+import json
 import logging
 
 from blinkpy.common.path_finder import PathFinder
@@ -37,13 +38,13 @@ class WPTServe(server_base.ServerBase):
         path_to_pywebsocket = finder.path_from_chromium_base('third_party', 'pywebsocket', 'src')
         path_to_wpt_support = finder.path_from_blink_tools('blinkpy', 'third_party', 'wpt')
         path_to_wpt_root = fs.join(path_to_wpt_support, 'wpt')
-        path_to_wpt_config = fs.join(path_to_wpt_support, 'wpt.config.json')
         path_to_wpt_tests = fs.abspath(fs.join(self._port_obj.layout_tests_dir(), 'external', 'wpt'))
         path_to_ws_handlers = fs.join(path_to_wpt_tests, 'websockets', 'handlers')
+        self._config_file = self._prepare_wptserve_config(path_to_wpt_support)
         wpt_script = fs.join(path_to_wpt_root, 'wpt')
         start_cmd = [self._port_obj.host.executable,
                      '-u', wpt_script, 'serve',
-                     '--config', path_to_wpt_config,
+                     '--config', self._config_file,
                      '--doc_root', path_to_wpt_tests]
 
         # TODO(burnik): Merge with default start_cmd once we roll in websockets.
@@ -59,15 +60,31 @@ class WPTServe(server_base.ServerBase):
 
         expiration_date = datetime.date(2025, 1, 4)
         if datetime.date.today() > expiration_date - datetime.timedelta(30):
-            logging.getLogger(__name__).error(
+            _log.error(
                 'Pre-generated keys and certificates are going to be expired at %s.'
-                ' Please re-generate them by following steps in %s/README.chromium.'
-                % (expiration_date.strftime('%b %d %Y'), path_to_wpt_support))
+                ' Please re-generate them by following steps in %s/README.chromium.',
+                expiration_date.strftime('%b %d %Y'), path_to_wpt_support)
+
+    def _prepare_wptserve_config(self, path_to_wpt_support):
+        fs = self._filesystem
+        template_path = fs.join(path_to_wpt_support, 'wpt.config.json')
+        config = json.loads(fs.read_text_file(template_path))
+        config['aliases'].append({
+            'url-path': '/gen/',
+            'local-dir': self._port_obj.generated_sources_directory()
+        })
+
+        f, temp_file = fs.open_text_tempfile('.json')
+        json.dump(config, f)
+        f.close()
+        return temp_file
 
     def _stop_running_server(self):
         self._wait_for_action(self._check_and_kill_wptserve)
         if self._filesystem.exists(self._pid_file):
             self._filesystem.remove(self._pid_file)
+        if self._filesystem.exists(self._config_file):
+            self._filesystem.remove(self._config_file)
 
     def _check_and_kill_wptserve(self):
         """Tries to kill wptserve.

third_party/blink/tools/blinkpy/web_tests/servers/wptserve_unittest.py

@@ -2,6 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import json
 import logging
 
 from blinkpy.common.system.log_testing import LoggingTestCase
@@ -12,11 +13,19 @@ from blinkpy.web_tests.servers.wptserve import WPTServe
 
 class TestWPTServe(LoggingTestCase):
 
+    def setUp(self):
+        super(TestWPTServe, self).setUp()
+        self.host = MockHost()
+        self.port = test.TestPort(self.host)
+        self.host.filesystem.write_text_file(
+            '/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt.config.json',
+            '{"ports": {}, "aliases": []}'
+        )
+
     # pylint: disable=protected-access
 
     def test_init_start_cmd(self):
-        test_port = test.TestPort(MockHost())
-        server = WPTServe(test_port, '/foo')
+        server = WPTServe(self.port, '/foo')
         self.assertEqual(
             server._start_cmd,  # pylint: disable=protected-access
             [
@@ -25,14 +34,22 @@ class TestWPTServe(LoggingTestCase):
                 '/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt/wpt',
                 'serve',
                 '--config',
-                '/mock-checkout/third_party/blink/tools/blinkpy/third_party/wpt/wpt.config.json',
+                server._config_file,
                 '--doc_root',
                 '/test.checkout/LayoutTests/external/wpt'
             ])
 
+    def test_init_gen_config(self):
+        server = WPTServe(self.port, '/foo')
+        config = json.loads(self.port._filesystem.read_text_file(server._config_file))
+        self.assertEqual(len(config['aliases']), 1)
+        self.assertDictEqual(
+            config['aliases'][0],
+            {'url-path': '/gen/', 'local-dir': '/mock-checkout/out/Release/gen'}
+        )
+
     def test_init_env(self):
-        test_port = test.TestPort(MockHost())
-        server = WPTServe(test_port, '/foo')
+        server = WPTServe(self.port, '/foo')
         self.assertEqual(
             server._env,  # pylint: disable=protected-access
             {
@@ -45,23 +62,21 @@ class TestWPTServe(LoggingTestCase):
         # Allow asserting about debug logs.
         self.set_logging_level(logging.DEBUG)
 
-        host = MockHost()
-        test_port = test.TestPort(host)
-        host.filesystem.write_text_file('/log_file_dir/access_log', 'foo')
-        host.filesystem.write_text_file('/log_file_dir/error_log', 'foo')
-        host.filesystem.write_text_file('/tmp/pidfile', '7')
+        self.host.filesystem.write_text_file('/log_file_dir/access_log', 'foo')
+        self.host.filesystem.write_text_file('/log_file_dir/error_log', 'foo')
+        self.host.filesystem.write_text_file('/tmp/pidfile', '7')
 
-        server = WPTServe(test_port, '/log_file_dir')
+        server = WPTServe(self.port, '/log_file_dir')
         server._pid_file = '/tmp/pidfile'
         server._spawn_process = lambda: 4
         server._is_server_running_on_all_ports = lambda: True
 
         # Simulate a process that never gets killed.
-        host.executive.check_running_pid = lambda _: True
+        self.host.executive.check_running_pid = lambda _: True
 
         server.start()
         self.assertEqual(server._pid, 4)
-        self.assertIsNone(host.filesystem.files[server._pid_file])
+        self.assertIsNone(self.host.filesystem.files[server._pid_file])
 
         # In this case, we'll try to kill the process repeatedly,
         # then give up and just try to start a new process anyway.
@@ -79,3 +94,9 @@ class TestWPTServe(LoggingTestCase):
                 'DEBUG: all ports are available\n',
                 'DEBUG: wptserve successfully started (pid = 4)\n'
             ])
+
+    def test_stop_running_server_removes_temp_files(self):
+        server = WPTServe(self.port, '/foo')
+        server._stop_running_server()
+        self.assertFalse(self.host.filesystem.exists(server._pid_file))
+        self.assertFalse(self.host.filesystem.exists(server._config_file))