build/toolchain/win/midl.py: Python 3 support

* Use bytestrings for operations in cases where files are read in binary mode.
* Output from subprocess is a bytestring in Python 3. Need to decode it before using in normal string operations.
* Due to use of bytestrings need to encode UUID string.
* Had to use branching code for ord() due to string vs bytestring in Py2/3.
* array() typecode 'c' (char) has been removed in Python 3. Use 'B' (unsigned char) instead.
* Use '//' for division where necessary. Add 'from __future__ import division' to avoid this in the future.
* Import reduce() for Python 3.
* Fixed open(mode='U') DeprecationWarnings. Use io.open for universal newlines on both Py2/3.

The changes are backwards compatible with Python 2.
Formatting changes are a result of running "git cl format --python".

Errors fixed:
Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 240, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 205, in main
    processing = set(os.path.basename(x)
  File "../../build/toolchain/win/midl.py", line 206, in <genexpr>
    for x in lines if x.startswith(prefixes))
TypeError: a bytes-like object is required, not 'str'

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 239, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 213, in main
    ZapTimestamp(os.path.join(tmp_dir, f))
  File "../../build/toolchain/win/midl.py", line 71, in ZapTimestamp
    contents = re.sub(
  File "C:\Program Files\Python38\lib\re.py", line 208, in sub
    return _compile(pattern, flags).sub(repl, string, count)
TypeError: cannot use a string pattern on a bytes-like object

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 241, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 167, in main
    overwrite_cls_guid(os.path.join(outdir, h),
  File "../../build/toolchain/win/midl.py", line 152, in overwrite_cls_guid
    overwrite_cls_guid_h(h_file, dynamic_guid)
  File "../../build/toolchain/win/midl.py", line 99, in overwrite_cls_guid_h
    br'class DECLSPEC_UUID("%s")' % str(dynamic_guid), contents)
TypeError: %b requires a bytes-like object, or an object that implements __bytes__, not 'str'

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 241, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 167, in main
    overwrite_cls_guid(os.path.join(outdir, h),
  File "../../build/toolchain/win/midl.py", line 153, in overwrite_cls_guid
    overwrite_cls_guid_iid(iid_file, dynamic_guid)
  File "../../build/toolchain/win/midl.py", line 106, in overwrite_cls_guid_iid
    hexuuid += ','.join('0x%02x' % ord(b) for b in dynamic_guid.bytes[8:])
  File "../../build/toolchain/win/midl.py", line 106, in <genexpr>
    hexuuid += ','.join('0x%02x' % ord(b) for b in dynamic_guid.bytes[8:])
TypeError: ord() expected string of length 1, but int found

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 241, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 167, in main
    overwrite_cls_guid(os.path.join(outdir, h),
  File "../../build/toolchain/win/midl.py", line 154, in overwrite_cls_guid
    overwrite_cls_guid_tlb(tlb_file, dynamic_guid)
  File "../../build/toolchain/win/midl.py", line 120, in overwrite_cls_guid_tlb
    assert ord(contents[type_off]) == 0x25, "expected coclass"
TypeError: ord() expected string of length 1, but int found

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 241, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 167, in main
    overwrite_cls_guid(os.path.join(outdir, h),
  File "../../build/toolchain/win/midl.py", line 154, in overwrite_cls_guid
    overwrite_cls_guid_tlb(tlb_file, dynamic_guid)
  File "../../build/toolchain/win/midl.py", line 125, in overwrite_cls_guid_tlb
    contents = array.array('c', contents)
ValueError: bad typecode (must be b, B, u, h, H, i, I, l, L, q, Q, f or d)

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 241, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 167, in main
    overwrite_cls_guid(os.path.join(outdir, h),
  File "../../build/toolchain/win/midl.py", line 154, in overwrite_cls_guid
    overwrite_cls_guid_tlb(tlb_file, dynamic_guid)
  File "../../build/toolchain/win/midl.py", line 131, in overwrite_cls_guid_tlb
    hashtab = [0xffffffff] * (0x80 / 4)
TypeError: can't multiply sequence by non-int of type 'float'

Traceback (most recent call last):
  File "../../build/toolchain/win/midl.py", line 242, in <module>
    sys.exit(main(*sys.argv[1:]))
  File "../../build/toolchain/win/midl.py", line 168, in main
    overwrite_cls_guid(os.path.join(outdir, h),
  File "../../build/toolchain/win/midl.py", line 155, in overwrite_cls_guid
    overwrite_cls_guid_tlb(tlb_file, dynamic_guid)
  File "../../build/toolchain/win/midl.py", line 138, in overwrite_cls_guid_tlb
    guidhash = reduce(operator.xor, [w for w in words]) % (0x80 // 4)
NameError: name 'reduce' is not defined

../../build/toolchain/win/midl.py:231: DeprecationWarning: 'U' mode is deprecated
  open(fromfile, 'bU').readlines(),
../../build/toolchain/win/midl.py:232: DeprecationWarning: 'U' mode is deprecated
  open(tofile, 'bU').readlines(), fromfile, tofile)))

Bug: 941669
Change-Id: I3e5985c732db114eff2209e428ec9ee3f209e3b6
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1829205
Commit-Queue: Raul Tambre <raul@tambre.ee>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Auto-Submit: Raul Tambre <raul@tambre.ee>
Cr-Commit-Position: refs/heads/master@{#705426}

build/toolchain/win/midl.py

@@ -2,12 +2,14 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from __future__ import division
 from __future__ import print_function
 
 import array
 import difflib
 import distutils.dir_util
 import filecmp
+import io
 import operator
 import os
 import re
@@ -18,6 +20,8 @@ import sys
 import tempfile
 import uuid
 
+from functools import reduce
+
 
 def ZapTimestamp(filename):
   contents = open(filename, 'rb').read()
@@ -45,65 +49,75 @@ def ZapTimestamp(filename):
     # There might be more custom data after that, but these 3 blocks are always
     # there for file-level metadata.
     # All data is little-endian in the file.
-    assert contents[0:8] == 'MSFT\x02\x00\x01\x00'
+    assert contents[0:8] == b'MSFT\x02\x00\x01\x00'
     ntypes, = struct.unpack_from('<I', contents, 0x20)
     custom_off, custom_len = struct.unpack_from(
         '<II', contents, 0x54 + 4*ntypes + 11*16)
     assert custom_len >= 0x54
     # First: Type string (0x8), followed by 0x3e characters.
-    assert contents[custom_off:custom_off+6] == '\x08\x00\x3e\x00\x00\x00'
+    assert contents[custom_off:custom_off + 6] == b'\x08\x00\x3e\x00\x00\x00'
     assert re.match(
-        r'Created by MIDL version 8\.\d\d\.\d{4} at ... Jan 1. ..:..:.. 2038\n',
+        br'Created by MIDL version 8\.\d\d\.\d{4} at ... Jan 1. ..:..:.. 2038\n',
         contents[custom_off + 6:custom_off + 6 + 0x3e])
     # Second: Type uint32 (0x13) storing 0x7fffffff (followed by WW / 0x57 pad)
     assert contents[custom_off+6+0x3e:custom_off+6+0x3e+8] == \
-        '\x13\x00\xff\xff\xff\x7f\x57\x57'
+        b'\x13\x00\xff\xff\xff\x7f\x57\x57'
     # Third: Type uint32 (0x13) storing MIDL compiler version.
-    assert contents[custom_off+6+0x3e+8:custom_off+6+0x3e+8+2] == '\x13\x00'
+    assert contents[custom_off + 6 + 0x3e + 8:custom_off + 6 + 0x3e + 8 +
+                    2] == b'\x13\x00'
     # Replace "Created by" string with fixed string, and fixed MIDL version with
     # 8.1.622 always.
-    contents = (contents[0:custom_off+6] +
-        'Created by MIDL version 8.xx.xxxx at a redacted point in time\n' +
+    contents = (
+        contents[0:custom_off + 6] +
+        b'Created by MIDL version 8.xx.xxxx at a redacted point in time\n' +
         # uint32 (0x13) val 0x7fffffff, WW, uint32 (0x13), val 0x0801026e, WW
-        '\x13\x00\xff\xff\xff\x7f\x57\x57\x13\x00\x6e\x02\x01\x08\x57\x57' +
+        b'\x13\x00\xff\xff\xff\x7f\x57\x57\x13\x00\x6e\x02\x01\x08\x57\x57' +
         contents[custom_off + 0x54:])
   else:
     contents = re.sub(
-        r'File created by MIDL compiler version 8\.\d\d\.\d{4} \*/\r\n'
-        r'/\* at ... Jan 1. ..:..:.. 2038',
-        r'File created by MIDL compiler version 8.xx.xxxx */\r\n'
-        r'/* at a redacted point in time', contents)
+        br'File created by MIDL compiler version 8\.\d\d\.\d{4} \*/\r\n'
+        br'/\* at ... Jan 1. ..:..:.. 2038',
+        br'File created by MIDL compiler version 8.xx.xxxx */\r\n'
+        br'/* at a redacted point in time', contents)
     contents = re.sub(
-        r'    Oicf, W1, Zp8, env=(.....) \(32b run\), '
-        r'target_arch=(AMD64|X86) 8\.\d\d\.\d{4}',
-        r'    Oicf, W1, Zp8, env=\1 (32b run), target_arch=\2 8.xx.xxxx',
+        br'    Oicf, W1, Zp8, env=(.....) \(32b run\), '
+        br'target_arch=(AMD64|X86) 8\.\d\d\.\d{4}',
+        br'    Oicf, W1, Zp8, env=\1 (32b run), target_arch=\2 8.xx.xxxx',
         contents)
     # TODO(thakis): If we need more hacks than these, try to verify checked-in
     # outputs when we're using the hermetic toolchain.
     # midl.exe older than 8.1.622 omit '//' after #endif, fix that:
-    contents = contents.replace('#endif !_MIDL_USE_GUIDDEF_',
-                                '#endif // !_MIDL_USE_GUIDDEF_')
+    contents = contents.replace(b'#endif !_MIDL_USE_GUIDDEF_',
+                                b'#endif // !_MIDL_USE_GUIDDEF_')
     # midl.exe puts the midl version into code in one place.  To have
     # predictable output, lie about the midl version if it's not 8.1.622.
     # This is unfortunate, but remember that there's beauty too in imperfection.
-    contents = contents.replace('0x801026c, /* MIDL Version 8.1.620 */',
-                                '0x801026e, /* MIDL Version 8.1.622 */')
+    contents = contents.replace(b'0x801026c, /* MIDL Version 8.1.620 */',
+                                b'0x801026e, /* MIDL Version 8.1.622 */')
   open(filename, 'wb').write(contents)
 
 
 def overwrite_cls_guid_h(h_file, dynamic_guid):
   contents = open(h_file, 'rb').read()
-  contents = re.sub(r'class DECLSPEC_UUID\("[^"]*"\)',
-                    r'class DECLSPEC_UUID("%s")' % str(dynamic_guid), contents)
+  contents = re.sub(br'class DECLSPEC_UUID\("[^"]*"\)',
+                    br'class DECLSPEC_UUID("%s")' % str(dynamic_guid).encode(),
+                    contents)
   open(h_file, 'wb').write(contents)
 
 
 def overwrite_cls_guid_iid(iid_file, dynamic_guid):
   contents = open(iid_file, 'rb').read()
   hexuuid = '0x%08x,0x%04x,0x%04x,' % dynamic_guid.fields[0:3]
-  hexuuid += ','.join('0x%02x' % ord(b) for b in dynamic_guid.bytes[8:])
-  contents = re.sub(r'MIDL_DEFINE_GUID\(CLSID, ([^,]*),[^)]*\)',
-                    r'MIDL_DEFINE_GUID(CLSID, \1,%s)' % hexuuid, contents)
+
+  # dynamic_guid.bytes is a bytestring in Py3, but a normal string in Py2.
+  if sys.version_info.major == 2:
+    hexuuid += ','.join('0x%02x' % ord(b) for b in dynamic_guid.bytes[8:])
+  else:
+    hexuuid += ','.join('0x%02x' % b for b in dynamic_guid.bytes[8:])
+
+  contents = re.sub(br'MIDL_DEFINE_GUID\(CLSID, ([^,]*),[^)]*\)',
+                    br'MIDL_DEFINE_GUID(CLSID, \1,%s)' % hexuuid.encode(),
+                    contents)
   open(iid_file, 'wb').write(contents)
 
 
@@ -112,27 +126,34 @@ def overwrite_cls_guid_tlb(tlb_file, dynamic_guid):
   # section contains type descriptions, and the first type should be our
   # coclass.  It points to the type's GUID in section 6, the GUID section.
   contents = open(tlb_file, 'rb').read()
-  assert contents[0:8] == 'MSFT\x02\x00\x01\x00'
+  assert contents[0:8] == b'MSFT\x02\x00\x01\x00'
   ntypes, = struct.unpack_from('<I', contents, 0x20)
   type_off, type_len = struct.unpack_from('<II', contents, 0x54 + 4*ntypes)
-  assert ord(contents[type_off]) == 0x25, "expected coclass"
+
+  # contents is a bytestring in Python 3, but a normal string in Py2.
+  if sys.version_info.major == 2:
+    coclass = ord(contents[type_off])
+  else:
+    coclass = contents[type_off]
+  assert coclass == 0x25, "expected coclass"
+
   guidind = struct.unpack_from('<I', contents, type_off + 0x2c)[0]
   guid_off, guid_len = struct.unpack_from(
       '<II', contents, 0x54 + 4*ntypes + 5*16)
   assert guidind + 14 <= guid_len
-  contents = array.array('c', contents)
+  contents = array.array('B', contents)
   struct.pack_into('<IHH8s', contents, guid_off + guidind,
                    *(dynamic_guid.fields[0:3] + (dynamic_guid.bytes[8:],)))
   # The GUID is correct now, but there's also a GUID hashtable in section 5.
   # Need to recreate that too.  Since the hash table uses chaining, it's
   # easiest to recompute it from scratch rather than trying to patch it up.
-  hashtab = [0xffffffff] * (0x80 / 4)
+  hashtab = [0xffffffff] * (0x80 // 4)
   for guidind in range(guid_off, guid_off + guid_len, 24):
     guidbytes, typeoff, nextguid = struct.unpack_from(
         '<16sII', contents, guidind)
     words = struct.unpack('<8H', guidbytes)
     # midl seems to use the following simple hash function for GUIDs:
-    guidhash = reduce(operator.xor, [w for w in words]) % (0x80 / 4)
+    guidhash = reduce(operator.xor, [w for w in words]) % (0x80 // 4)
     nextguid = hashtab[guidhash]
     struct.pack_into('<I', contents, guidind + 0x14, nextguid)
     hashtab[guidhash] = guidind - guid_off
@@ -199,7 +220,7 @@ def main(arch, gendir, outdir, dynamic_guid, tlb, h, dlldata, iid, proxy, idl,
     # to filter is pairs of lines that look like this:
     # Processing C:\Program Files (x86)\Microsoft SDKs\...\include\objidl.idl
     # objidl.idl
-    lines = out.splitlines()
+    lines = out.decode('utf-8').splitlines()
     prefixes = ('Processing ', '64 bit Processing ')
     processing = set(os.path.basename(x)
                      for x in lines if x.startswith(prefixes))
@@ -223,8 +244,8 @@ def main(arch, gendir, outdir, dynamic_guid, tlb, h, dlldata, iid, proxy, idl,
         tofile = os.path.join(tmp_dir, f)
         print(''.join(
             difflib.unified_diff(
-                open(fromfile, 'U').readlines(),
-                open(tofile, 'U').readlines(), fromfile, tofile)))
+                io.open(fromfile).readlines(),
+                io.open(tofile).readlines(), fromfile, tofile)))
       delete_tmp_dir = False
       print('To rebaseline:')
       print(r'  copy /y %s\* %s' % (tmp_dir, source))