Report the value of concurrent_links from "gn gen" when on bots

Hoping that this will aid in debugging out of memory failures.

Bug: 1113246
Change-Id: Ibf2baf859598a2af2d08d911c82d3c65f2249d28
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2328190
Commit-Queue: Andrew Grieve <agrieve@chromium.org>
Reviewed-by: Mohamed Heikal <mheikal@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Cr-Commit-Position: refs/heads/master@{#795975}

BUILD.gn

@@ -12,6 +12,7 @@ import("//build/config/compiler/compiler.gni")
 import("//build/config/features.gni")
 import("//build/config/sanitizers/sanitizers.gni")
 import("//build/config/ui.gni")
+import("//build/gn_logs.gni")
 import("//build/util/generate_wrapper.gni")
 import("//chrome/browser/buildflags.gni")
 import("//chrome/browser/media/router/features.gni")
@@ -757,8 +758,8 @@ group("gn_all") {
     }
   }
 
-  if (((is_linux || is_chromeos) && !is_chromecast) || (is_win && use_libfuzzer) ||
-      (use_libfuzzer && is_mac)) {
+  if (((is_linux || is_chromeos) && !is_chromecast) ||
+      (is_win && use_libfuzzer) || (use_libfuzzer && is_mac)) {
     deps += [
       "//chrome/services/ipp_parser/public/cpp:fuzzers",
       "//testing/libfuzzer/fuzzers",
@@ -1398,3 +1399,15 @@ assert(
                      "*\bwin/*",
                    ]) != [],
     "Do not use a platform name in your output directory (found \"$root_build_dir\"). http://crbug.com/548283")
+
+# Write debug logs to gn_logs.txt.
+_lines = [
+           "Generated during 'gn gen' by //BUILD.gn.",
+           "",
+         ] + build_gn_logs
+
+# GN evaluates each .gn file once per toolchain, so restricting to default
+# toolchain will ensure write_file() is called only once.
+assert(current_toolchain == default_toolchain)
+
+write_file("$root_build_dir/gn_logs.txt", _lines)

build/gn_logs.gni

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//build/toolchain/concurrent_links.gni")
+
+# Log lines for gn_logs.txt that originate from within //build.
+build_gn_logs = [ "#### get_concurrent_links.py ####" ] + concurrent_links_logs

build/toolchain/concurrent_links.gni

@@ -70,5 +70,11 @@ if (concurrent_links == -1) {
 
   # TODO(crbug.com/617429) Pass more build configuration info to the script
   # so that we can compute better values.
-  concurrent_links = exec_script("get_concurrent_links.py", _args, "value")
+  _command_dict = exec_script("get_concurrent_links.py", _args, "scope")
+
+  concurrent_links = _command_dict.concurrent_links
+  concurrent_links_logs = _command_dict.explanation
+} else {
+  concurrent_links_logs =
+      [ "concurrent_links set by GN arg (value=$concurrent_links)" ]
 }

build/toolchain/get_concurrent_links.py

+#!/usr/bin/env python
 # Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -7,28 +8,32 @@
 
 from __future__ import print_function
 
+import argparse
 import multiprocessing
-import optparse
 import os
 import re
 import subprocess
 import sys
 
+sys.path.insert(1, os.path.join(os.path.dirname(__file__), '..'))
+import gn_helpers
+
+
 def _GetTotalMemoryInBytes():
   if sys.platform in ('win32', 'cygwin'):
     import ctypes
 
     class MEMORYSTATUSEX(ctypes.Structure):
       _fields_ = [
-        ("dwLength", ctypes.c_ulong),
-        ("dwMemoryLoad", ctypes.c_ulong),
-        ("ullTotalPhys", ctypes.c_ulonglong),
-        ("ullAvailPhys", ctypes.c_ulonglong),
-        ("ullTotalPageFile", ctypes.c_ulonglong),
-        ("ullAvailPageFile", ctypes.c_ulonglong),
-        ("ullTotalVirtual", ctypes.c_ulonglong),
-        ("ullAvailVirtual", ctypes.c_ulonglong),
-        ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
+          ("dwLength", ctypes.c_ulong),
+          ("dwMemoryLoad", ctypes.c_ulong),
+          ("ullTotalPhys", ctypes.c_ulonglong),
+          ("ullAvailPhys", ctypes.c_ulonglong),
+          ("ullTotalPageFile", ctypes.c_ulonglong),
+          ("ullAvailPageFile", ctypes.c_ulonglong),
+          ("ullTotalVirtual", ctypes.c_ulonglong),
+          ("ullAvailVirtual", ctypes.c_ulonglong),
+          ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
       ]
 
     stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
@@ -53,34 +58,55 @@ def _GetTotalMemoryInBytes():
 
 
 def _GetDefaultConcurrentLinks(mem_per_link_gb, reserve_mem_gb):
+  explanation = []
+  explanation.append('mem_per_link_gb={} reserve_mem_gb={}'.format(
+      mem_per_link_gb, reserve_mem_gb))
   # Inherit the legacy environment variable for people that have set it in GYP.
-  pool_size = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
-  if pool_size:
-    return pool_size
+  num_links = int(os.getenv('GYP_LINK_CONCURRENCY', 0))
+  if num_links:
+    reason = 'GYP_LINK_CONCURRENCY'
+  else:
+    mem_total_bytes = _GetTotalMemoryInBytes()
+    mem_total_bytes = max(0, mem_total_bytes - reserve_mem_gb * 2**30)
+    mem_cap = int(max(1, mem_total_bytes / mem_per_link_gb / 2**30))
+    hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
+
+    try:
+      cpu_cap = multiprocessing.cpu_count()
+    except:
+      cpu_cap = 1
 
-  mem_total_bytes = _GetTotalMemoryInBytes()
-  mem_total_bytes = max(0, mem_total_bytes - reserve_mem_gb * 2**30)
-  num_concurrent_links = int(max(1, mem_total_bytes / mem_per_link_gb / 2**30))
-  hard_cap = max(1, int(os.getenv('GYP_LINK_CONCURRENCY_MAX', 2**32)))
+    explanation.append('cpu_count={} mem_total_bytes={:.1f}GiB'.format(
+        cpu_cap, mem_total_bytes / 2**30))
 
-  try:
-    cpu_cap = multiprocessing.cpu_count()
-  except:
-    cpu_cap = 1
+    num_links = min(mem_cap, hard_cap, cpu_cap)
+    if num_links == cpu_cap:
+      reason = 'cpu_count'
+    elif num_links == hard_cap:
+      reason = 'GYP_LINK_CONCURRENCY_MAX'
+    else:
+      reason = 'RAM'
 
-  return min(num_concurrent_links, hard_cap, cpu_cap)
+  explanation.append('concurrent_links={}  (reason: {})'.format(
+      num_links, reason))
+  return num_links, explanation
 
 
 def main():
-  parser = optparse.OptionParser()
-  parser.add_option('--mem_per_link_gb', action="store", type="int", default=8)
-  parser.add_option('--reserve_mem_gb', action="store", type="int", default=0)
-  parser.disable_interspersed_args()
-  options, _ = parser.parse_args()
-
-  print(_GetDefaultConcurrentLinks(options.mem_per_link_gb,
-                                   options.reserve_mem_gb))
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--mem_per_link_gb', type=int, default=8)
+  parser.add_argument('--reserve_mem_gb', type=int, default=0)
+  options = parser.parse_args()
+
+  num_links, explanation = _GetDefaultConcurrentLinks(options.mem_per_link_gb,
+                                                      options.reserve_mem_gb)
+  sys.stdout.write(
+      gn_helpers.ToGNString({
+          'concurrent_links': num_links,
+          'explanation': explanation,
+      }))
   return 0
 
+
 if __name__ == '__main__':
   sys.exit(main())