Add --target_os=... support to run_tool.py and generate_compdb.py

To consume win32-specific cmdline parameters (e.g. -imsvc, /FI or /Yu)
clang needs to be invoked with `--driver-mode=cl`.  This CL ensures that
the driver-mode cmdline switch is appended not only when running a
rewriter tool on Windows, but also when run_tool.py or
generate_compdb.py are invoked with an explicit --target_os=win cmdline
argument.

Manual tests:

1. Manually run unit tests for tools/clang/pylib/clang/compile_db.py:
    $ pushd tools/clang/pylib/clang/
    $ python compile_db_test.py

2. Manually tried running the rewriter on linux, when targeting win:
    $ cat out/rewrite-win/args.gn
    clang_use_chrome_plugins = false
    target_os = "win"
    $ tools/clang/scripts/run_tool.py --tool rewrite_raw_ptr_fields \
        --generate-compdb --target_os=win -p out/rewrite-win \
        content/browser/renderer_host/render_process_host_impl.cc \
        >~/scratch/rewriter.out

3. Manually tried running generate_compdb.py:
    $ tools/clang/scripts/generate_compdb.py -p out/rewrite gn_all \
        -o ~/scratch/compdb

Bug: 1069567
Change-Id: I54fcee164d7bb05119c8267b43e0d872bc2f436b
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2481745
Commit-Queue: Łukasz Anforowicz <lukasza@chromium.org>
Reviewed-by: Daniel Cheng <dcheng@chromium.org>
Reviewed-by: Nico Weber <thakis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#818687}

tools/clang/pylib/clang/compile_db.py

@@ -19,7 +19,17 @@ _CMD_LINE_RE = re.compile(
 _debugging = False
 
 
-def _ProcessCommand(command):
+def _IsTargettingWindows(target_os):
+  if target_os is not None:
+    # Available choices are based on: gn help target_os
+    assert target_os in [
+        'android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'
+    ]
+    return target_os == 'win'
+  return sys.platform == 'win32'
+
+
+def _ProcessCommand(command, target_os):
   """Removes gomacc(.exe). On Windows inserts --driver-mode=cl as the first arg.
 
   Note that we deliberately don't use shlex.split here, because it doesn't work
@@ -34,7 +44,7 @@ def _ProcessCommand(command):
   # 9.0.0.
   driver_mode = ''
   # Only specify for Windows. Other platforms do fine without it.
-  if sys.platform == 'win32' and '--driver_mode' not in command:
+  if _IsTargettingWindows(target_os) and '--driver-mode' not in command:
     driver_mode = '--driver-mode=cl'
 
   match = _CMD_LINE_RE.search(command)
@@ -57,9 +67,9 @@ def _ProcessCommand(command):
   return " ".join(command_parts)
 
 
-def _ProcessEntry(entry):
+def _ProcessEntry(entry, target_os):
   """Transforms one entry in a Windows compile db to be clang-tool friendly."""
-  entry['command'] = _ProcessCommand(entry['command'])
+  entry['command'] = _ProcessCommand(entry['command'], target_os)
 
   # Expand the contents of the response file, if any.
   # http://llvm.org/bugs/show_bug.cgi?id=21634
@@ -79,7 +89,7 @@ def _ProcessEntry(entry):
   return entry
 
 
-def ProcessCompileDatabaseIfNeeded(compile_db):
+def ProcessCompileDatabaseIfNeeded(compile_db, target_os=None):
   """Make the compile db generated by ninja on Windows more clang-tool friendly.
 
   Args:
@@ -88,12 +98,9 @@ def ProcessCompileDatabaseIfNeeded(compile_db):
   Returns:
     A postprocessed compile db that clang tooling can use.
   """
-  compile_db = [_ProcessEntry(e) for e in compile_db]
+  compile_db = [_ProcessEntry(e, target_os) for e in compile_db]
 
-  # TODO(dcheng): Ideally this would check target_os... but not sure there's an
-  # easy way to do that, and (for now) cross-compiles don't work without custom
-  # patches anyway.
-  if sys.platform != 'win32':
+  if not _IsTargettingWindows(target_os):
     return compile_db
 
   if _debugging:

tools/clang/pylib/clang/compile_db_test.py

@@ -68,7 +68,7 @@ class CompileDbTest(unittest.TestCase):
     except AttributeError:
       self.assertItemsEqual(processed_compile_db, _TEST_COMPILE_DB)
 
-  def testProcessForWindows(self):
+  def testProcessForWindows_HostPlatformBased(self):
     sys.platform = 'win32'
     processed_compile_db = compile_db.ProcessCompileDatabaseIfNeeded(
         _TEST_COMPILE_DB)
@@ -77,6 +77,15 @@ class CompileDbTest(unittest.TestCase):
     for actual, expected in zip(processed_compile_db, _EXPECTED_COMPILE_DB):
       self.assertDictEqual(actual, expected)
 
+  def testProcessForWindows_TargetOsBased(self):
+    sys.platform = 'linux2'
+    processed_compile_db = compile_db.ProcessCompileDatabaseIfNeeded(
+        _TEST_COMPILE_DB, target_os='win')
+
+    # Check each entry individually to improve readability of the output.
+    for actual, expected in zip(processed_compile_db, _EXPECTED_COMPILE_DB):
+      self.assertDictEqual(actual, expected)
+
 
 if __name__ == '__main__':
   unittest.main()

tools/clang/scripts/generate_compdb.py

@@ -32,16 +32,20 @@ def main(argv):
       'targets',
       nargs='*',
       help='Additional targets to pass to ninja')
+  parser.add_argument(
+      '--target_os',
+      choices=['android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'],
+      help='Target OS - see `gn help target_os`. Set to "win" when ' +
+      'cross-compiling Windows from Linux or another host')
   parser.add_argument(
       '-o',
       help='File to write the compilation database to. Defaults to stdout')
 
   args = parser.parse_args()
 
-  compdb_text = json.dumps(
-      compile_db.ProcessCompileDatabaseIfNeeded(
-          compile_db.GenerateWithNinja(args.p, args.targets)),
-      indent=2)
+  compdb_text = json.dumps(compile_db.ProcessCompileDatabaseIfNeeded(
+      compile_db.GenerateWithNinja(args.p, args.targets), args.target_os),
+                           indent=2)
   if args.o is None:
     print(compdb_text)
   else:

tools/clang/scripts/run_tool.py

@@ -157,7 +157,8 @@ def _GetEntriesFromCompileDB(build_directory, source_filenames):
   ]
 
 
-def _UpdateCompileCommandsIfNeeded(compile_commands, files_list):
+def _UpdateCompileCommandsIfNeeded(compile_commands, files_list,
+                                   target_os=None):
   """ Filters compile database to only include required files, and makes it
   more clang-tool friendly on Windows.
 
@@ -179,7 +180,8 @@ def _UpdateCompileCommandsIfNeeded(compile_commands, files_list):
   else:
     filtered_compile_commands = compile_commands
 
-  return compile_db.ProcessCompileDatabaseIfNeeded(filtered_compile_commands)
+  return compile_db.ProcessCompileDatabaseIfNeeded(filtered_compile_commands,
+                                                   target_os)
 
 
 def _ExecuteTool(toolname, tool_args, build_directory, compdb_entry):
@@ -339,6 +341,11 @@ def main():
       '-p',
       required=True,
       help='path to the directory that contains the compile database')
+  parser.add_argument(
+      '--target_os',
+      choices=['android', 'chromeos', 'ios', 'linux', 'nacl', 'mac', 'win'],
+      help='Target OS - see `gn help target_os`. Set to "win" when ' +
+      'cross-compiling Windows from Linux or another host')
   parser.add_argument(
       'path_filter',
       nargs='*',
@@ -375,8 +382,9 @@ def main():
 
   if args.generate_compdb:
     compile_commands = compile_db.GenerateWithNinja(args.p)
-    compile_commands = _UpdateCompileCommandsIfNeeded(
-        compile_commands, source_filenames)
+    compile_commands = _UpdateCompileCommandsIfNeeded(compile_commands,
+                                                      source_filenames,
+                                                      args.target_os)
     with open(os.path.join(args.p, 'compile_commands.json'), 'w') as f:
       f.write(json.dumps(compile_commands, indent=2))