Update blink web_tests to use pywebsocket3

This commit updates blink web_tests to use pywebsocket3. It updates
blinkpy/server/pywebsocket to import pywebsocket3 for its path.

This commit also fixes the websocket handlers in web_tests. It update
the handlers to use pywebsocket3 functions instead of old ones, and also
updates the websocket handlers that use chr() to generate bytes string,
to use util.pack_byte(), since chr() does not generate bytes in
python 3.

Change-Id: I2fbc365571b2ca0ee947355a2f95700350675fb6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2059705Reviewed-by: Kentaro Hara <haraken@chromium.org>
Reviewed-by: Yutaka Hirano <yhirano@chromium.org>
Reviewed-by: Adam Rice <ricea@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Keita Suzuki <suzukikeita@google.com>
Cr-Commit-Position: refs/heads/master@{#748556}

BUILD.gn

@@ -1006,6 +1006,7 @@ if (!is_ios) {
         "//third_party/blink/web_tests/VirtualTestSuites",
         "//third_party/blink/web_tests/WebDriverExpectations",
         "//third_party/pywebsocket/src/mod_pywebsocket/",
+        "//third_party/pywebsocket3/src/mod_pywebsocket/",
       ]
       data_deps = [
         "//chrome:chrome",
@@ -1121,6 +1122,7 @@ if (!is_ios) {
       "//third_party/blink/tools/",
       "//third_party/blink/web_tests/",
       "//third_party/pywebsocket/src/mod_pywebsocket/",
+      "//third_party/pywebsocket3/src/mod_pywebsocket/",
     ]
 
     if (is_win) {

chrome/credential_provider/test/BUILD.gn

@@ -48,7 +48,7 @@ test("gcp_unittests") {
 
   data = [
     "//net/tools/testserver/",
-    "//third_party/pywebsocket/src/mod_pywebsocket/",
+    "//third_party/pywebsocket3/src/mod_pywebsocket/",
     "//third_party/tlslite/",
   ]
 }

third_party/blink/tools/blinkpy/web_tests/servers/pywebsocket.py

@@ -56,7 +56,7 @@ class PyWebSocket(server_base.ServerBase):
         time_str = time.strftime('%d%b%Y-%H%M%S')
         log_file_name = _WS_LOG_PREFIX + time_str
         self._error_log = self._filesystem.join(self._output_dir, log_file_name + '-err.txt')
-        pywebsocket_base = PathFinder(self._filesystem).path_from_chromium_base('third_party', 'pywebsocket', 'src')
+        pywebsocket_base = PathFinder(self._filesystem).path_from_chromium_base('third_party', 'pywebsocket3', 'src')
         pywebsocket_script = self._filesystem.join(pywebsocket_base, 'mod_pywebsocket', 'standalone.py')
 
         self._start_cmd = [

third_party/blink/web_tests/http/tests/websocket/bad-handshake-crash_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -8,7 +8,7 @@ def web_socket_do_extra_handshake(request):
            b'Connection: Upgrade\r\n'
            b'Sec-WebSocket-Accept: %s\r\n'
            b'\xa5:\r\n'
-           b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+           b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
 
     request.connection.write(msg)
 

third_party/blink/web_tests/http/tests/websocket/binary-frames_wsh.py

 from mod_pywebsocket import common
 from mod_pywebsocket import stream
+from mod_pywebsocket import util
+from six.moves import range
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,7 +9,7 @@ def web_socket_do_extra_handshake(request):
 
 
 def web_socket_transfer_data(request):
-    messages_to_send = ['Hello, world!', '', all_distinct_bytes()]
+    messages_to_send = [b'Hello, world!', b'', all_distinct_bytes()]
     for message in messages_to_send:
         # FIXME: Should use better API to send binary messages when pywebsocket
         # supports it.
@@ -17,4 +19,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return b''.join([util.pack_byte(i) for i in range(256)])

third_party/blink/web_tests/http/tests/websocket/check-binary-messages_wsh.py

+import six
+from six.moves import range
+
 from mod_pywebsocket import common
 from mod_pywebsocket import msgutil
+from mod_pywebsocket import util
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,11 +11,11 @@ def web_socket_do_extra_handshake(request):
 
 
 def web_socket_transfer_data(request):
-    expected_messages = ['Hello, world!', '', all_distinct_bytes()]
+    expected_messages = [b'Hello, world!', b'', all_distinct_bytes()]
 
     for test_number, expected_message in enumerate(expected_messages):
         message = msgutil.receive_message(request)
-        if type(message) == str and message == expected_message:
+        if type(message) == six.binary_type and message == expected_message:
             msgutil.send_message(request, 'PASS: Message #%d.' % test_number)
         else:
             msgutil.send_message(
@@ -20,4 +24,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return b''.join([util.pack_byte(i) for i in range(256)])

third_party/blink/web_tests/http/tests/websocket/duplicated-headers_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -11,7 +11,7 @@ def web_socket_do_extra_handshake(request):
                b'foo: hoge\r\n'
                b'FOO: FUGA\r\n'
                b'xxx: yyy\r\n'
-               b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+               b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
 
     request.connection.write(message)
     # Prevents pywebsocket from sending its own handshake message.

third_party/blink/web_tests/http/tests/websocket/fragmented-binary-frames_wsh.py

+from six.moves import range
+
 from mod_pywebsocket import common
 from mod_pywebsocket import stream
+from mod_pywebsocket import util
 
 
 def web_socket_do_extra_handshake(request):
@@ -8,10 +11,8 @@ def web_socket_do_extra_handshake(request):
 
 def web_socket_transfer_data(request):
     # pyformat: disable
-    messages_to_send = [['Hello, ', 'world!'],
-                        ['', 'Hello, ', '', 'world!', ''],
-                        ['', '', ''],
-                        [chr(i) for i in xrange(256)]]
+    messages_to_send = [[b'Hello, ', b'world!'], [b'', b'Hello, ', b'', b'world!', b''], [b'', b'', b''],
+                        [util.pack_byte(i) for i in range(256)]]
     # pyformat: enable
     for message_list in messages_to_send:
         for index, message in enumerate(message_list):

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-extensions-header_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -8,7 +8,7 @@ def web_socket_do_extra_handshake(request):
                b'Connection: Upgrade\r\n'
                b'Sec-WebSocket-Accept: %s\r\n'
                b'Sec-WebSocket-Extensions: x-foo\r\n'
-               b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+               b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
 
     request.connection.write(message)
     # Prevents pywebsocket from sending its own handshake message.

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-maxlength_wsh.py

@@ -20,7 +20,7 @@
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -31,7 +31,7 @@ def web_socket_do_extra_handshake(request):
            b'Upgrade: websocket\r\n'
            b'Connection: Upgrade\r\n'
            b'Sec-WebSocket-Accept: %s\r\n'
-           b'\r\n') % (b'.' * 1024, compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0])
+           b'\r\n') % (b'.' * 1024, compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key']))
 
     request.connection.write(msg)
     # Prevents pywebsocket from sending its own handshake message.

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-mismatch-protocol-header_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -8,7 +8,7 @@ def web_socket_do_extra_handshake(request):
            b'Connection: Upgrade\r\n'
            b'Sec-WebSocket-Accept: %s\r\n'
            b'Sec-WebSocket-Protocol: MismatchProtocol\r\n'
-           b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+           b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
 
     request.connection.write(msg)
     # Prevents pywebsocket from sending its own handshake message.

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-more-accept-header_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -8,7 +8,7 @@ def web_socket_do_extra_handshake(request):
            b'Connection: Upgrade\r\n'
            b'Sec-WebSocket-Accept: %s\r\n'
            b'Sec-WebSocket-Accept: XXXXthisiswrongXXXX\r\n'
-           b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+           b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
 
     request.connection.write(msg)
     # Prevents pywebsocket from sending its own handshake message.

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-more-protocol-header_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -9,7 +9,7 @@ def web_socket_do_extra_handshake(request):
            b'Sec-WebSocket-Accept: %s\r\n'
            b'Sec-WebSocket-Protocol: MatchProtocol\r\n'
            b'Sec-WebSocket-Protocol: MismatchProtocol\r\n'
-           b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+           b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
     request.connection.write(msg)
     # Prevents pywebsocket from sending its own handshake message.
     raise handshake.AbortedByUserException('Abort the connection')

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-no-connection-header_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,7 +7,7 @@ def web_socket_do_extra_handshake(request):
     msg = (b'HTTP/1.1 101 Switching Protocols\r\n'
            b'Upgrade: websocket\r\n'
            b'Sec-WebSocket-Accept: %s\r\n'
-           b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+           b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
     request.connection.write(msg)
     # Prevents pywebsocket from sending its own handshake message.
     raise handshake.AbortedByUserException('Abort the connection')

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-no-upgrade-header_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,7 +7,7 @@ def web_socket_do_extra_handshake(request):
     msg = (b'HTTP/1.1 101 Switching Protocols\r\n'
            b'Connection: Upgrade\r\n'
            b'Sec-WebSocket-Accept: %s\r\n'
-           b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+           b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
     request.connection.write(msg)
     # Prevents pywebsocket from sending its own handshake message.
     raise handshake.AbortedByUserException('Abort the connection')

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-null-subprotocol-selection_wsh.py

 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,7 +7,7 @@ def web_socket_do_extra_handshake(request):
                b'Upgrade: websocket\r\n'
                b'Connection: Upgrade\r\n'
                b'Sec-WebSocket-Accept: %s\r\n'
-               b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+               b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
     request.connection.write(message)
     # Prevents pywebsocket from sending its own handshake message.
     raise handshake.AbortedByUserException('Abort the connection')

third_party/blink/web_tests/http/tests/websocket/handshake-fail-by-prepended-null_wsh.py

@@ -22,7 +22,7 @@
 from __future__ import division
 import time
 from mod_pywebsocket import stream
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -38,7 +38,7 @@ def web_socket_do_extra_handshake(request):
             b'Upgrade: websocket\r\n'
             b'Connection: Upgrade\r\n'
             b'Sec-WebSocket-Accept: %s\r\n'
-            b'\r\n') % compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0]
+            b'\r\n') % compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key'])
     request.connection.write(msg)
     # continue writing data until the client disconnects
     while True:

third_party/blink/web_tests/http/tests/websocket/invalid-encode-length_wsh.py

 import re
 import struct
 from mod_pywebsocket import common
+from mod_pywebsocket import msgutil
+from mod_pywebsocket import util
 
 
 def web_socket_do_extra_handshake(request):
@@ -20,9 +22,9 @@ def web_socket_transfer_data(request):
 
     # pywebsocket refuses to create a frame with error encode length.
     # Thus, we need to build a frame manually.
-    header = chr(0x80 | common.OPCODE_TEXT)  # 0x80 is for "fin" bit.
+    header = util.pack_byte(0x80 | common.OPCODE_TEXT)  # 0x80 is for "fin" bit.
     # No Mask and two bytes extended payload length.
-    header += chr(payload_length)
+    header += util.pack_byte(payload_length)
     if payload_length == 126:
         header += struct.pack('!H', extended_length)
     elif payload_length == 127:
@@ -32,4 +34,4 @@ def web_socket_transfer_data(request):
                              'FAIL: Query value is incorrect or missing')
         return
     request.connection.write(header)
-    request.connection.write('X' * extended_length)
+    request.connection.write(b'X' * extended_length)

third_party/blink/web_tests/http/tests/websocket/permessage-deflate-invalid-parameter_wsh.py

@@ -28,7 +28,7 @@
 
 from six.moves.urllib import parse
 from mod_pywebsocket import handshake
-from mod_pywebsocket.handshake.hybi import compute_accept
+from mod_pywebsocket.handshake.hybi import compute_accept_from_unicode
 
 
 def web_socket_do_extra_handshake(request):
@@ -43,7 +43,7 @@ def web_socket_do_extra_handshake(request):
                b'Connection: Upgrade\r\n'
                b'Sec-WebSocket-Accept: %s\r\n'
                b'Sec-WebSocket-Extensions: permessage-deflate'
-               b'%s\r\n') % (compute_accept(request.headers_in['Sec-WebSocket-Key'].encode('UTF-8'))[0], message_parameters)
+               b'%s\r\n') % (compute_accept_from_unicode(request.headers_in['Sec-WebSocket-Key']), message_parameters)
 
     request.connection.write(message)
     # Prevents pywebsocket from sending its own handshake message.

third_party/blink/web_tests/http/tests/websocket/send-file-blob_wsh.py

+from six.moves import range
+
 from mod_pywebsocket import common
 from mod_pywebsocket import msgutil
+from mod_pywebsocket import util
 
 
 def _retrieve_frame(stream):
@@ -17,7 +20,7 @@ def web_socket_do_extra_handshake(request):
 
 
 def web_socket_transfer_data(request):
-    expected_messages = ['Hello, world!']
+    expected_messages = [b'Hello, world!']
 
     for test_number, expected_message in enumerate(expected_messages):
         frame = _retrieve_frame(request.ws_stream)
@@ -33,4 +36,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return b''.join([util.pack_byte(i) for i in range(256)])

third_party/blink/web_tests/http/tests/websocket/too-long-payload_wsh.py

 import struct
 import time
 from mod_pywebsocket import common
+from mod_pywebsocket import util
 
 
 def web_socket_do_extra_handshake(request):
@@ -12,13 +13,13 @@ def web_socket_transfer_data(request):
 
     # pywebsocket refuses to send a frame with too long payload.
     # Thus, we need to build a frame manually.
-    header = chr(0x80 | common.OPCODE_TEXT)  # 0x80 is for "fin" bit.
-    header += chr(127)
+    header = util.pack_byte(0x80 | common.OPCODE_TEXT)  # 0x80 is for "fin" bit.
+    header += util.pack_byte(127)
     header += struct.pack('!Q', length)
     request.connection.write(header)
 
     # Send data indefinitely to simulate a real (broken) server sending a big
     # frame. A client should ignore these bytes and abort the connection.
     while True:
-        request.connection.write('X' * 4096)
+        request.connection.write(b'X' * 4096)
         time.sleep(1)

third_party/blink/web_tests/http/tests/websocket/workers/resources/binary-frames_wsh.py

+from six.moves import range
+
 from mod_pywebsocket import common
 from mod_pywebsocket import stream
+from mod_pywebsocket import util
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,7 +10,7 @@ def web_socket_do_extra_handshake(request):
 
 
 def web_socket_transfer_data(request):
-    messages_to_send = ['Hello, world!', '', all_distinct_bytes()]
+    messages_to_send = [b'Hello, world!', b'', all_distinct_bytes()]
     for message in messages_to_send:
         # FIXME: Should use better API to send binary messages when pywebsocket
         # supports it.
@@ -17,4 +20,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return b''.join([util.pack_byte(i) for i in range(256)])

third_party/blink/web_tests/http/tests/websocket/workers/resources/check-binary-messages_wsh.py

+import six
+from six.moves import range
+
 from mod_pywebsocket import common
 from mod_pywebsocket import msgutil
+from mod_pywebsocket import util
 
 
 def web_socket_do_extra_handshake(request):
@@ -7,11 +11,11 @@ def web_socket_do_extra_handshake(request):
 
 
 def web_socket_transfer_data(request):
-    expected_messages = ['Hello, world!', '', all_distinct_bytes()]
+    expected_messages = [b'Hello, world!', b'', all_distinct_bytes()]
 
     for test_number, expected_message in enumerate(expected_messages):
         message = msgutil.receive_message(request)
-        if type(message) == str and message == expected_message:
+        if isinstance(message, six.binary_type) and message == expected_message:
             msgutil.send_message(request, 'PASS: Message #%d.' % test_number)
         else:
             msgutil.send_message(
@@ -20,4 +24,4 @@ def web_socket_transfer_data(request):
 
 
 def all_distinct_bytes():
-    return ''.join([chr(i) for i in xrange(256)])
+    return b''.join([util.pack_byte(i) for i in range(256)])