goma_link: add --allowlist option

During testing and troubleshooting, it is often useful to have
goma_link.py and goma_ld.py perform distributed ThinLTO for targets
that are not on the allow list. This change adds an --allowlist option
which, when given, causes the scripts to proceed as if the target
being built is on the allowlist, regardless of whether it actually is.

Bug: 1072888
Change-Id: Ide4f273b022d3451cc2eb5341a3ee5e0be1ff323
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2275633
Commit-Queue: Bob Haarman <inglorion@chromium.org>
Reviewed-by: George Burgess <gbiv@chromium.org>
Cr-Commit-Position: refs/heads/master@{#784209}

tools/clang/scripts/goma_ld.py

@@ -45,10 +45,11 @@ class GomaLinkUnix(goma_link.GomaLinkBase):
     # For other targets, we fall back to local ThinLTO. We must use ThinLTO
     # because we build with -fsplit-lto-unit, which requires code generation
     # be done for each object and target.
-    if args.output is None or os.path.basename(
-        args.output) not in self.ALLOWLIST:
-      # Returning None causes the original, non-distributed linker command to be
-      # invoked.
+    # Returning None from this function causes the original, non-distributed
+    # linker command to be invoked.
+    if args.output is None:
+      return None
+    if not (args.allowlist or os.path.basename(args.output) in self.ALLOWLIST):
       return None
     return super(GomaLinkUnix, self).analyze_args(args, *posargs, **kwargs)
 

tools/clang/scripts/goma_link.py

@@ -187,6 +187,9 @@ def parse_args(args):
   ap.add_argument('--gomacc', help='path to gomacc.')
   ap.add_argument('--jobs', '-j', help='maximum number of concurrent jobs.')
   ap.add_argument('--no-gomacc', action='store_true', help='do not use gomacc.')
+  ap.add_argument('--allowlist',
+                  action='store_true',
+                  help='act as if the target is on the allow list.')
   try:
     splitpos = args.index('--')
   except:
@@ -641,7 +644,7 @@ class GomaLinkBase(object):
     # Currently, we don't detect this situation. We could, but it might
     # be better to instead move this logic out of this script and into
     # the build system.
-    use_common_objects = basename not in self.ALLOWLIST
+    use_common_objects = not (args.allowlist or basename in self.ALLOWLIST)
     common_dir = 'common_objs'
     gen_dir = 'lto.' + basename
     params = self.analyze_args(args, gen_dir, common_dir, use_common_objects)

tools/clang/scripts/goma_link_integration_tests.py

@@ -183,6 +183,41 @@ class GomaLinkIntegrationTest(unittest.TestCase):
       self.assertNotIn(b'jmp', disasm)
       self.assertNotIn(b'call', disasm)
 
+  def test_override_allowlist(self):
+    with named_directory() as d, working_directory(d):
+      _create_inputs(d)
+      os.makedirs('obj')
+      subprocess.check_call([
+          self.clangcl(), '-c', '-O2', '-flto=thin', 'main.cpp',
+          '-Foobj/main.obj'
+      ])
+      subprocess.check_call([
+          self.clangcl(), '-c', '-O2', '-flto=thin', 'foo.cpp', '-Foobj/foo.obj'
+      ])
+      rc = goma_link.GomaLinkWindows().main([
+          'goma_link.py', '--generate', '--allowlist', '--',
+          self.lld_link(), '-nodefaultlib', '-entry:main', '-opt:lldlto=2',
+          '-out:main.exe', 'obj/main.obj', 'obj/foo.obj'
+      ])
+      # Should succeed.
+      self.assertEqual(rc, 0)
+      # Check that we have rules for main and foo, and that they are
+      # not common objects.
+      with open(os.path.join(d, 'lto.main.exe', 'build.ninja')) as f:
+        buildrules = f.read()
+        codegen_match = re.search(r'^rule codegen\b.*?^[^ ]', buildrules,
+                                  re.MULTILINE | re.DOTALL)
+        self.assertIsNotNone(codegen_match)
+        codegen_text = codegen_match.group(0)
+        self.assertNotIn('-flto', codegen_text)
+        self.assertIn('build lto.main.exe/obj/main.obj.stamp : codegen ',
+                      buildrules)
+        self.assertIn('build lto.main.exe/obj/foo.obj.stamp : codegen ',
+                      buildrules)
+        link_match = re.search(r'^build main.exe : native-link\b.*?^[^ ]',
+                               buildrules, re.MULTILINE | re.DOTALL)
+        self.assertIsNotNone(link_match)
+
 
 class GomaLdIntegrationTest(unittest.TestCase):
   def clangxx(self):
@@ -477,6 +512,29 @@ class GomaLdIntegrationTest(unittest.TestCase):
             re.compile('^build [^:]+/main\\.o\\.stamp : codegen ',
                        re.MULTILINE))
 
+  def test_override_allowlist(self):
+    with named_directory() as d, working_directory(d):
+      _create_inputs(d)
+      subprocess.check_call([
+          self.clangxx(), '-c', '-Os', '-flto=thin', 'main.cpp', '-o', 'main.o'
+      ])
+      subprocess.check_call(
+          [self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o'])
+      rc = goma_ld.GomaLinkUnix().main([
+          'goma_ld.py', '--generate', '--allowlist', '--',
+          self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o',
+          'main'
+      ])
+      # Should succeed.
+      self.assertEqual(rc, 0)
+      # build.ninja file should have rules for main and foo.
+      ninjafile = os.path.join(d, 'lto.main', 'build.ninja')
+      self.assertTrue(os.path.exists(ninjafile))
+      with open(ninjafile) as f:
+        buildrules = f.read()
+        self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules)
+        self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules)
+
 
 if __name__ == '__main__':
   unittest.main()