Generate absl .def file at build time.

Instead of generating absl .def files at roll time, this CL switches
the build config to generate the required .def file on the fly while
building.

The advantages are:

1. No need to re-generate .def files when something external to absl
changes and affects .def files (e.g. when a new compiler optimization
it enabled).
2. No need to hard-code supported build flavors in the .def generator
script, resulting in multiple .def files checked-in.

On the other hand, this makes the build slightly slower than it
is today because reading all absl object files to generate the
.def file can take a few seconds.

Bug: 1046390
Change-Id: If3a8fb3dca0ed75dbd85655cff1263853dcc146a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2270094
Commit-Queue: Mirko Bonadei <mbonadei@chromium.org>
Reviewed-by: Bruce Dawson <brucedawson@chromium.org>
Cr-Commit-Position: refs/heads/master@{#824844}

third_party/abseil-cpp/BUILD.gn

@@ -16,6 +16,32 @@ config("absl_component_build") {
   defines = [ "ABSL_CONSUME_DLL" ]
 }
 
+action("generate_def_file") {
+  script = "//third_party/abseil-cpp/generate_def_file.py"
+
+  outputs = [ "$target_gen_dir/absl.def" ]
+
+  if (is_win) {
+    if (current_cpu == "x86") {
+      win_tool_arch = "environment.x86"
+    } else if (current_cpu == "x64") {
+      win_tool_arch = "environment.x64"
+    } else if (current_cpu == "arm64") {
+      win_tool_arch = "environment.arm64"
+    } else {
+      assert(false, "Need environment for this arch")
+    }
+  } else {
+    win_tool_arch = ""
+  }
+
+  args = [
+    rebase_path(outputs[0], root_build_dir),
+    win_tool_arch,
+  ]
+  deps = [ ":absl_component_deps" ]
+}
+
 # TODO(mbonadei): WebRTC tests and binaries use absl flags but they are
 # marked testonly because we don't want them to be usable in Chromium.
 # Add an absl_flags component which depends on the main absl component.
@@ -23,33 +49,9 @@ component("absl") {
   public_deps = [ ":absl_component_deps" ]
   if (is_component_build) {
     public_configs = [ ":absl_component_build" ]
-
-    if (is_win && is_clang) {
-      if (current_cpu == "x64") {
-        if (is_debug) {
-          sources = [ "symbols_x64_dbg.def" ]
-        } else {
-          if (is_asan) {
-            sources = [ "symbols_x64_rel_asan.def" ]
-          } else {
-            sources = [ "symbols_x64_rel.def" ]
-          }
-        }
-      }
-      if (current_cpu == "x86") {
-        if (is_debug) {
-          sources = [ "symbols_x86_dbg.def" ]
-        } else {
-          sources = [ "symbols_x86_rel.def" ]
-        }
-      }
-      if (current_cpu == "arm64") {
-        if (is_debug) {
-          sources = [ "symbols_arm64_dbg.def" ]
-        } else {
-          sources = [ "symbols_arm64_rel.def" ]
-        }
-      }
+    if (is_win) {
+      sources = get_target_outputs(":generate_def_file")
+      deps = [ ":generate_def_file" ]
     }
   }
 }
@@ -97,8 +99,6 @@ group("absl_component_deps") {
       "//third_party/abseil-cpp/absl/debugging:failure_signal_handler",
     ]
   }
-
-  visibility = [ ":absl" ]
 }
 
 group("default") {

third_party/abseil-cpp/generate_def_file.py

+#!/usr/bin/env python
+
+"""Script to generate Chromium's Abseil .def file from a GN action.
+
+Since Abseil doesn't export symbols, Chromium is forced to consider all
+Abseil's symbols as publicly visible. On POSIX it is possible to use
+-fvisibility=default but on Windows a .def file with all the symbols
+is needed.
+"""
+
+import fnmatch
+import logging
+import os
+import re
+import subprocess
+import sys
+import tempfile
+import time
+
+# Matches a mangled symbol that has 'absl' in it, this should be a good
+# enough heuristic to select Abseil symbols to list in the .def file.
+ABSL_SYM_RE = re.compile(r'0* [BT] (?P<symbol>(\?+)[^\?].*absl.*)')
+if sys.platform == 'win32':
+  # Typical dumpbin /symbol lines look like this:
+  # 04B 0000000C SECT14 notype       Static       | ?$S1@?1??SetCurrent
+  # ThreadIdentity@base_internal@absl@@YAXPAUThreadIdentity@12@P6AXPAX@Z@Z@4IA
+  #  (unsigned int `void __cdecl absl::base_internal::SetCurrentThreadIdentity...
+  # We need to start on "| ?" and end on the first " (" (stopping on space would
+  # also work).
+  # This regex is identical inside the () characters except for the ? after .*,
+  # which is needed to prevent greedily grabbing the undecorated version of the
+  # symbols.
+  ABSL_SYM_RE = '.*External     \| (?P<symbol>(\?+)[^\?].*?absl.*?) \(.*'
+  # Typical exported symbols in dumpbin /directives look like:
+  #    /EXPORT:?kHexChar@numbers_internal@absl@@3QBDB,DATA
+  ABSL_EXPORTED_RE = '.*/EXPORT:(.*),.*'
+
+
+def _GenerateDefFile(output_path, arch):
+  """Generates a .def file for the absl component build by reading absl object files."""
+  if sys.platform == 'win32':
+    env_pairs = open(arch).read()[:-2].split('\0')
+    env_dict = dict([item.split('=', 1) for item in env_pairs])
+    use_shell = True
+  else:
+    env_dict = None
+    use_shell = False
+
+  symbol_dumper = ['llvm-nm-9']
+  if sys.platform == 'win32':
+    symbol_dumper = ['dumpbin', '/symbols']
+
+  obj_files = []
+  for root, _dirnames, filenames in os.walk(
+      os.path.join('obj', 'third_party', 'abseil-cpp', 'absl')):
+    matched_files = fnmatch.filter(filenames, '*.obj')
+    obj_files.extend((os.path.join(root, f) for f in matched_files))
+
+
+  absl_symbols = set()
+  dll_exports = set()
+  if sys.platform == 'win32':
+    for f in obj_files:
+      # Track all of the functions exported with __declspec(dllexport) and
+      # don't list them in the .def file - double-exports are not allowed. The
+      # error is "lld-link: error: duplicate /export option".
+      exports_out = subprocess.check_output(['dumpbin', '/directives', f],
+                                            shell=use_shell,
+                                            env=env_dict,
+                                            cwd=os.getcwd())
+      for line in exports_out.splitlines():
+        line = line.decode('utf-8')
+        match = re.match(ABSL_EXPORTED_RE, line)
+        if match:
+          dll_exports.add(match.groups()[0])
+  for f in obj_files:
+    stdout = subprocess.check_output(symbol_dumper + [f],
+                                     shell=use_shell,
+                                     env=env_dict,
+                                     cwd=os.getcwd())
+    for line in stdout.splitlines():
+      try:
+        line = line.decode('utf-8')
+      except UnicodeDecodeError:
+        # Due to a dumpbin bug there are sometimes invalid utf-8 characters in
+        # the output. This only happens on an unimportant line so it can
+        # safely and silently be skipped.
+        # https://developercommunity.visualstudio.com/content/problem/1091330/dumpbin-symbols-produces-randomly-wrong-output-on.html
+        continue
+      match = re.match(ABSL_SYM_RE, line)
+      if match:
+        symbol = match.group('symbol')
+        assert symbol.count(' ') == 0, ('Regex matched too much, probably got '
+                                        'undecorated name as well')
+        # Avoid getting names exported with dllexport, to avoid
+        # "lld-link: error: duplicate /export option" on symbols such as:
+        # ?kHexChar@numbers_internal@absl@@3QBDB
+        if symbol in dll_exports:
+          continue
+        absl_symbols.add(symbol)
+
+  with open(output_path, 'w') as f:
+    f.write('EXPORTS\n')
+    for s in sorted(absl_symbols):
+      f.write('    {}\n'.format(s))
+
+
+if __name__ == '__main__':
+  logging.getLogger().setLevel(logging.INFO)
+  output_path = sys.argv[1]
+  if sys.platform == 'win32':
+    arch = sys.argv[2]
+  else:
+    arch = None
+
+  _GenerateDefFile(output_path, arch)

third_party/abseil-cpp/generate_def_files.py

-#!/usr/bin/env python
-
-# NOTE: This script requires python 3.
-
-"""Script to generate Chromium's Abseil .def files at roll time.
-
-This script generates //third_party/abseil-app/absl/symbols_*.def at Abseil
-roll time.
-
-Since Abseil doesn't export symbols, Chromium is forced to consider all
-Abseil's symbols as publicly visible. On POSIX it is possible to use
--fvisibility=default but on Windows a .def file with all the symbols
-is needed.
-
-Unless you are on a Windows machine, you need to set up your Chromium
-checkout for cross-compilation by following the instructions at
-https://chromium.googlesource.com/chromium/src.git/+/master/docs/win_cross.md.
-If you are on Windows, you may need to tweak this script to run, e.g. by
-changing "gn" to "gn.bat", changing "llvm-nm-9" to the name of your copy of
-llvm-nm, etc.
-"""
-
-import fnmatch
-import logging
-import os
-import re
-import subprocess
-import sys
-import tempfile
-import time
-
-# Matches a mangled symbol that has 'absl' in it, this should be a good
-# enough heuristic to select Abseil symbols to list in the .def file.
-ABSL_SYM_RE = re.compile(r'0* [BT] (?P<symbol>(\?+)[^\?].*absl.*)')
-if sys.platform == 'win32':
-  # Typical dumpbin /symbol lines look like this:
-  # 04B 0000000C SECT14 notype       Static       | ?$S1@?1??SetCurrent
-  # ThreadIdentity@base_internal@absl@@YAXPAUThreadIdentity@12@P6AXPAX@Z@Z@4IA
-  #  (unsigned int `void __cdecl absl::base_internal::SetCurrentThreadIdentity...
-  # We need to start on "| ?" and end on the first " (" (stopping on space would
-  # also work).
-  # This regex is identical inside the () characters except for the ? after .*,
-  # which is needed to prevent greedily grabbing the undecorated version of the
-  # symbols.
-  ABSL_SYM_RE = '.*External     \| (?P<symbol>(\?+)[^\?].*?absl.*?) \(.*'
-  # Typical exported symbols in dumpbin /directives look like:
-  #    /EXPORT:?kHexChar@numbers_internal@absl@@3QBDB,DATA
-  ABSL_EXPORTED_RE = '.*/EXPORT:(.*),.*'
-
-
-def _DebugOrRelease(is_debug):
-  return 'dbg' if is_debug else 'rel'
-
-
-def _GenerateDefFile(cpu, is_debug, extra_gn_args=[], suffix=None):
-  """Generates a .def file for the absl component build on the specified CPU."""
-  if extra_gn_args:
-    assert suffix != None, 'suffix is needed when extra_gn_args is used'
-
-  flavor = _DebugOrRelease(is_debug)
-  gn_args = [
-      'ffmpeg_branding = "Chrome"',
-      'is_component_build = true',
-      'is_debug = {}'.format(str(is_debug).lower()),
-      'proprietary_codecs = true',
-      'symbol_level = 0',
-      'target_cpu = "{}"'.format(cpu),
-      'target_os = "win"',
-  ]
-  gn_args.extend(extra_gn_args)
-
-  gn = 'gn'
-  autoninja = 'autoninja'
-  symbol_dumper = ['llvm-nm-9']
-  if sys.platform == 'win32':
-    gn = 'gn.bat'
-    autoninja = 'autoninja.bat'
-    symbol_dumper = ['dumpbin', '/symbols']
-    import shutil
-    if not shutil.which('dumpbin'):
-      logging.error('dumpbin not found. Run tools\win\setenv.bat.')
-      exit(1)
-  with tempfile.TemporaryDirectory() as out_dir:
-    logging.info('[%s - %s] Creating tmp out dir in %s', cpu, flavor, out_dir)
-    subprocess.check_call([gn, 'gen', out_dir, '--args=' + ' '.join(gn_args)],
-                          cwd=os.getcwd())
-    logging.info('[%s - %s] gn gen completed', cpu, flavor)
-    subprocess.check_call(
-        [autoninja, '-C', out_dir, 'third_party/abseil-cpp:absl_component_deps'],
-        cwd=os.getcwd())
-    logging.info('[%s - %s] autoninja completed', cpu, flavor)
-
-    obj_files = []
-    for root, _dirnames, filenames in os.walk(
-        os.path.join(out_dir, 'obj', 'third_party', 'abseil-cpp')):
-      matched_files = fnmatch.filter(filenames, '*.obj')
-      obj_files.extend((os.path.join(root, f) for f in matched_files))
-
-    logging.info('[%s - %s] Found %d object files.', cpu, flavor, len(obj_files))
-
-    absl_symbols = set()
-    dll_exports = set()
-    if sys.platform == 'win32':
-      for f in obj_files:
-        # Track all of the functions exported with __declspec(dllexport) and
-        # don't list them in the .def file - double-exports are not allowed. The
-        # error is "lld-link: error: duplicate /export option".
-        exports_out = subprocess.check_output(['dumpbin', '/directives', f], cwd=os.getcwd())
-        for line in exports_out.splitlines():
-          line = line.decode('utf-8')
-          match = re.match(ABSL_EXPORTED_RE, line)
-          if match:
-            dll_exports.add(match.groups()[0])
-    for f in obj_files:
-      stdout = subprocess.check_output(symbol_dumper + [f], cwd=os.getcwd())
-      for line in stdout.splitlines():
-        try:
-          line = line.decode('utf-8')
-        except UnicodeDecodeError:
-          # Due to a dumpbin bug there are sometimes invalid utf-8 characters in
-          # the output. This only happens on an unimportant line so it can
-          # safely and silently be skipped.
-          # https://developercommunity.visualstudio.com/content/problem/1091330/dumpbin-symbols-produces-randomly-wrong-output-on.html
-          continue
-        match = re.match(ABSL_SYM_RE, line)
-        if match:
-          symbol = match.group('symbol')
-          assert symbol.count(' ') == 0, ('Regex matched too much, probably got '
-                                          'undecorated name as well')
-          # Avoid getting names exported with dllexport, to avoid
-          # "lld-link: error: duplicate /export option" on symbols such as:
-          # ?kHexChar@numbers_internal@absl@@3QBDB
-          if symbol in dll_exports:
-            continue
-          absl_symbols.add(symbol)
-
-    logging.info('[%s - %s] Found %d absl symbols.', cpu, flavor, len(absl_symbols))
-
-    if extra_gn_args:
-      def_file = os.path.join('third_party', 'abseil-cpp',
-                              'symbols_{}_{}_{}.def'.format(cpu, flavor, suffix))
-    else:
-      def_file = os.path.join('third_party', 'abseil-cpp',
-                             'symbols_{}_{}.def'.format(cpu, flavor))
-
-    with open(def_file, 'w', newline='') as f:
-      f.write('EXPORTS\n')
-      for s in sorted(absl_symbols):
-        f.write('    {}\n'.format(s))
-
-    # Hack, it looks like there is a race in the directory cleanup.
-    time.sleep(10)
-
-  logging.info('[%s - %s] .def file successfully generated.', cpu, flavor)
-
-
-if __name__ == '__main__':
-  logging.getLogger().setLevel(logging.INFO)
-
-  if sys.version_info.major == 2:
-    logging.error('This script requires Python 3.')
-    exit(1)
-
-  if not os.getcwd().endswith('src') or not os.path.exists('chrome/browser'):
-    logging.error('Run this script from a chromium/src/ directory.')
-    exit(1)
-
-  _GenerateDefFile('x86', True)
-  _GenerateDefFile('x86', False)
-  _GenerateDefFile('x64', True)
-  _GenerateDefFile('x64', False)
-  _GenerateDefFile('x64', False, ['is_asan = true'], 'asan')
-  _GenerateDefFile('arm64', True)
-  _GenerateDefFile('arm64', False)