Revert of Make landmines work on local builds too (patchset #3 of...

Revert of Make landmines work on local builds too (patchset #3 of https://codereview.chromium.org/457003004/)

Reason for revert:
Apparently this requires win_toolchain.json to exist, but I'm not sure how it's supposed to get there (as seen on a clobber build):


Traceback (most recent call last):
  File "src/build/landmines.py", line 132, in <module>
    sys.exit(main())
  File "src/build/landmines.py", line 119, in main
    gyp_environment.SetEnvironment()
  File "C:\b\build\slave\win_trunk\build\src\build\gyp_environment.py", line 33, in SetEnvironment
    vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
  File "C:\b\build\slave\win_trunk\build\src\build\vs_toolchain.py", line 33, in SetEnvironmentAndGetRuntimeDllDirs
    with open(json_data_file, 'r') as tempf:
IOError: [Errno 2] No such file or directory: 'C:\\b\\build\\slave\\win_trunk\\build\\src\\build\\win_toolchain.json'

Original issue's description:
> Make landmines work on local builds too
> 
> Moves (some of) gyp environment setup out of gyp_chromium into separate
> module, and shares that between gyp_chromium and landmines.py.
> 
> landmines.py is added as the first entry in DEPS hooks so that it can
> clobber the entire build directory before running other hooks that
> extract/generate into the build dir.
> 
> R=iannucci@chromium.org
> BUG=400011
> 
> Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=289099

TBR=scottmg@chromium.org
NOTREECHECKS=true
NOTRY=true
BUG=400011

Review URL: https://codereview.chromium.org/469623002

Cr-Commit-Position: refs/heads/master@{#289158}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@289158 0039d316-1c4b-4281-b951-d872f2087c98

.gitignore

@@ -29,7 +29,6 @@
 .cproject
 .gdb_history
 .gdbinit
-.landmines
 .metadata
 .project
 .pydevproject

DEPS

@@ -586,17 +586,6 @@ skip_child_includes = [
 
 
 hooks = [
-  {
-    # This clobbers when necessary (based on get_landmines.py). It must be the
-    # first hook so that other things that get/generate into the output
-    # directory will not subsequently be clobbered.
-    "name": "landmines",
-    "pattern": ".",
-    "action": [
-        "python",
-        "src/build/landmines.py",
-    ],
-  },
   {
     # This downloads binaries for Native Client's newlib toolchain.
     # Done in lieu of building the toolchain from scratch as it can take

build/get_landmines.py

@@ -8,6 +8,7 @@ This file emits the list of reasons why a particular build needs to be clobbered
 (or a list of 'landmines').
 """
 
+import optparse
 import sys
 
 import landmine_utils
@@ -20,9 +21,10 @@ gyp_msvs_version = landmine_utils.gyp_msvs_version
 platform = landmine_utils.platform
 
 
-def print_landmines():
+def print_landmines(target):
   """
   ALL LANDMINES ARE EMITTED FROM HERE.
+  target can be one of {'Release', 'Debug', 'Debug_x64', 'Release_x64'}.
   """
   if (distributor() == 'goma' and platform() == 'win32' and
       builder() == 'ninja'):
@@ -58,7 +60,16 @@ def print_landmines():
 
 
 def main():
-  print_landmines()
+  parser = optparse.OptionParser()
+  parser.add_option('-t', '--target',
+                    help=='Target for which the landmines have to be emitted')
+
+  options, args = parser.parse_args()
+
+  if args:
+    parser.error('Unknown arguments %s' % args)
+
+  print_landmines(options.target)
   return 0
 
 

build/gyp_chromium

@@ -8,7 +8,7 @@
 # is invoked by Chromium beyond what can be done in the gclient hooks.
 
 import glob
-import gyp_environment
+import gyp_helper
 import os
 import re
 import shlex
@@ -197,6 +197,10 @@ if __name__ == '__main__':
     args.append('-Ganalyzer_output_path=' + args.pop(0))
 
   if int(os.environ.get('GYP_CHROMIUM_NO_ACTION', 0)):
+    # Check for landmines (reasons to clobber the build) in any case.
+    print 'Running build/landmines.py...'
+    subprocess.check_call(
+        [sys.executable, os.path.join(script_dir, 'landmines.py')])
     print 'Skipping gyp_chromium due to GYP_CHROMIUM_NO_ACTION env var.'
     sys.exit(0)
 
@@ -221,6 +225,8 @@ if __name__ == '__main__':
     p.communicate()
     sys.exit(p.returncode)
 
+  gyp_helper.apply_chromium_gyp_env()
+
   # This could give false positives since it doesn't actually do real option
   # parsing.  Oh well.
   gyp_file_specified = False
@@ -240,8 +246,6 @@ if __name__ == '__main__':
     else:
       args.append(os.path.join(script_dir, 'all.gyp'))
 
-  gyp_environment.SetEnvironment()
-
   # There shouldn't be a circular dependency relationship between .gyp files,
   # but in Chromium's .gyp files, on non-Mac platforms, circular relationships
   # currently exist.  The check for circular dependencies is currently
@@ -260,6 +264,21 @@ if __name__ == '__main__':
     print 'Error: make gyp generator not supported (check GYP_GENERATORS).'
     sys.exit(1)
 
+  # Default to ninja on linux and windows, but only if no generator has
+  # explicitly been set.
+  # Also default to ninja on mac, but only when not building chrome/ios.
+  # . -f / --format has precedence over the env var, no need to check for it
+  # . set the env var only if it hasn't been set yet
+  # . chromium.gyp_env has been applied to os.environ at this point already
+  if sys.platform.startswith(('linux', 'win', 'freebsd')) and \
+      not os.environ.get('GYP_GENERATORS'):
+    os.environ['GYP_GENERATORS'] = 'ninja'
+  elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \
+      not 'OS=ios' in os.environ.get('GYP_DEFINES', []):
+    os.environ['GYP_GENERATORS'] = 'ninja'
+
+  vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
+
   # If CHROMIUM_GYP_SYNTAX_CHECK is set to 1, it will invoke gyp with --check
   # to enfore syntax checking.
   syntax_check = os.environ.get('CHROMIUM_GYP_SYNTAX_CHECK')
@@ -303,7 +322,13 @@ if __name__ == '__main__':
   gyp_rc = gyp.main(args)
 
   if not use_analyzer:
-    vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
+    # Check for landmines (reasons to clobber the build). This must be run here,
+    # rather than a separate runhooks step so that any environment modifications
+    # from above are picked up.
+    print 'Running build/landmines.py...'
+    subprocess.check_call(
+      [sys.executable, os.path.join(script_dir, 'landmines.py')])
+
     if vs2013_runtime_dll_dirs:
       x64_runtime, x86_runtime = vs2013_runtime_dll_dirs
       vs_toolchain.CopyVsRuntimeDlls(

build/gyp_environment.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""
-Sets up various automatic gyp environment variables. These are used by
-gyp_chromium and landmines.py which run at different stages of runhooks. To
-make sure settings are consistent between them, all setup should happen here.
-"""
-
-import gyp_helper
-import os
-import sys
-import vs_toolchain
-
-def SetEnvironment():
-  """Sets defaults for GYP_* variables."""
-  gyp_helper.apply_chromium_gyp_env()
-
-  # Default to ninja on linux and windows, but only if no generator has
-  # explicitly been set.
-  # Also default to ninja on mac, but only when not building chrome/ios.
-  # . -f / --format has precedence over the env var, no need to check for it
-  # . set the env var only if it hasn't been set yet
-  # . chromium.gyp_env has been applied to os.environ at this point already
-  if sys.platform.startswith(('linux', 'win', 'freebsd')) and \
-      not os.environ.get('GYP_GENERATORS'):
-    os.environ['GYP_GENERATORS'] = 'ninja'
-  elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \
-      not 'OS=ios' in os.environ.get('GYP_DEFINES', []):
-    os.environ['GYP_GENERATORS'] = 'ninja'
-
-  vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()

build/landmines.py

@@ -4,8 +4,10 @@
 # found in the LICENSE file.
 
 """
-This script runs every build as the first hook (See DEPS). If it detects that
-the build should be clobbered, it will remove the build directory.
+This script runs every build as a hook. If it detects that the build should
+be clobbered, it will touch the file <build_dir>/.landmine_triggered. The
+various build scripts will then check for the presence of this file and clobber
+accordingly. The script will also emit the reasons for the clobber to stdout.
 
 A landmine is tripped when a builder checks out a different revision, and the
 diff between the new landmines and the old ones is non-null. At this point, the
@@ -14,11 +16,9 @@ build is clobbered.
 
 import difflib
 import errno
-import gyp_environment
 import logging
 import optparse
 import os
-import shutil
 import sys
 import subprocess
 import time
@@ -29,32 +29,35 @@ import landmine_utils
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 
 
-def get_build_dir(build_tool, is_iphone=False):
+def get_target_build_dir(build_tool, target, is_iphone=False):
   """
   Returns output directory absolute path dependent on build and targets.
   Examples:
-    r'c:\b\build\slave\win\build\src\out'
-    '/mnt/data/b/build/slave/linux/build/src/out'
-    '/b/build/slave/ios_rel_device/build/src/xcodebuild'
+    r'c:\b\build\slave\win\build\src\out\Release'
+    '/mnt/data/b/build/slave/linux/build/src/out/Debug'
+    '/b/build/slave/ios_rel_device/build/src/xcodebuild/Release-iphoneos'
 
   Keep this function in sync with tools/build/scripts/slave/compile.py
   """
   ret = None
   if build_tool == 'xcode':
-    ret = os.path.join(SRC_DIR, 'xcodebuild')
+    ret = os.path.join(SRC_DIR, 'xcodebuild',
+        target + ('-iphoneos' if is_iphone else ''))
   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
-    ret = os.path.join(SRC_DIR, 'out')
+    ret = os.path.join(SRC_DIR, 'out', target)
   elif build_tool in ['msvs', 'vs', 'ib']:
-    ret = os.path.join(SRC_DIR, 'build')
+    ret = os.path.join(SRC_DIR, 'build', target)
   else:
     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
   return os.path.abspath(ret)
 
 
-def clobber_if_necessary(new_landmines):
+def set_up_landmines(target, new_landmines):
   """Does the work of setting, planting, and triggering landmines."""
-  out_dir = get_build_dir(landmine_utils.builder())
-  landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
+  out_dir = get_target_build_dir(landmine_utils.builder(), target,
+                                 landmine_utils.platform() == 'ios')
+
+  landmines_path = os.path.join(out_dir, '.landmines')
   try:
     os.makedirs(out_dir)
   except OSError as e:
@@ -62,6 +65,7 @@ def clobber_if_necessary(new_landmines):
       pass
 
   if os.path.exists(landmines_path):
+    triggered = os.path.join(out_dir, '.landmines_triggered')
     with open(landmines_path, 'r') as f:
       old_landmines = f.readlines()
     if old_landmines != new_landmines:
@@ -69,13 +73,12 @@ def clobber_if_necessary(new_landmines):
       diff = difflib.unified_diff(old_landmines, new_landmines,
           fromfile='old_landmines', tofile='new_landmines',
           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
-      sys.stdout.write('Clobbering due to:\n')
-      sys.stdout.writelines(diff)
-
-      # Clobber.
-      shutil.rmtree(out_dir)
 
-  # Save current set of landmines for next time.
+      with open(triggered, 'w') as f:
+        f.writelines(diff)
+    elif os.path.exists(triggered):
+      # Remove false triggered landmines.
+      os.remove(triggered)
   with open(landmines_path, 'w') as f:
     f.writelines(new_landmines)
 
@@ -116,14 +119,14 @@ def main():
   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
     return 0
 
-  gyp_environment.SetEnvironment()
-
-  landmines = []
-  for s in landmine_scripts:
-    proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
-    output, _ = proc.communicate()
-    landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
-  clobber_if_necessary(landmines)
+  for target in ('Debug', 'Release', 'Debug_x64', 'Release_x64'):
+    landmines = []
+    for s in landmine_scripts:
+      proc = subprocess.Popen([sys.executable, s, '-t', target],
+                              stdout=subprocess.PIPE)
+      output, _ = proc.communicate()
+      landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
+    set_up_landmines(target, landmines)
 
   return 0