Trigger build request using git try instead of try_job_http on bisect bots

Bisect builders fail to apply patches sent via try_job_http,
because of this we were not able to build revisions with DEPS patches.
Now we are using "git try" instead to trigger build request on bisect builders.

BUG=411418
NOTRY=true

Review URL: https://codereview.chromium.org/607803005

Cr-Commit-Position: refs/heads/master@{#297918}

tools/auto_bisect/bisect_perf_regression_test.py

@@ -5,12 +5,18 @@
 import os
 import re
 import shutil
+import sys
 import unittest
 
+SRC = os.path.join(os.path.dirname(__file__), os.path.pardir, os.path.pardir)
+sys.path.append(os.path.join(SRC, 'third_party', 'pymock'))
+
 import bisect_perf_regression
 import bisect_results
+import mock
 import source_control as source_control_module
 
+
 def _GetBisectPerformanceMetricsInstance():
   """Returns an instance of the BisectPerformanceMetrics class."""
   options_dict = {
@@ -348,5 +354,156 @@ class DepotDirectoryRegistryTest(unittest.TestCase):
     self.assertEqual(self.cur_dir, '/mock/src/foo')
 
 
+# The tests below test private functions (W0212).
+# pylint: disable=W0212
+class GitTryJobTestCases(unittest.TestCase):
+  """Test case for bisect try job."""
+  def setUp(self):
+    bisect_utils_patcher = mock.patch('bisect_perf_regression.bisect_utils')
+    self.mock_bisect_utils = bisect_utils_patcher.start()
+    self.addCleanup(bisect_utils_patcher.stop)
+
+  def _SetupRunGitMock(self, git_cmds):
+    """Setup RunGit mock with expected output for given git command."""
+    def side_effect(git_cmd_args):
+      for val in git_cmds:
+        if set(val[0]) == set(git_cmd_args):
+          return val[1]
+    self.mock_bisect_utils.RunGit = mock.Mock(side_effect=side_effect)
+
+  def _AssertRunGitExceptions(self, git_cmds, func, *args):
+    """Setup RunGit mock and tests RunGitException.
+
+      Args:
+        git_cmds: List of tuples with git command and expected output.
+        func: Callback function to be executed.
+        args: List of arguments to be passed to the function.
+    """
+    self._SetupRunGitMock(git_cmds)
+    self.assertRaises(bisect_perf_regression.RunGitError,
+                      func,
+                      *args)
+
+  def testNotGitRepo(self):
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    cmds = [(['rev-parse', '--abbrev-ref', 'HEAD'], (None, 128))]
+    self._AssertRunGitExceptions(cmds,
+                                 bisect_perf_regression._PrepareBisectBranch,
+                                 parent_branch, new_branch)
+
+  def testFailedCheckoutMaster(self):
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    cmds = [
+        (['rev-parse', '--abbrev-ref', 'HEAD'], (new_branch, 0)),
+        (['checkout', '-f', parent_branch], ('Checkout Failed', 1))]
+    self._AssertRunGitExceptions(cmds,
+                                 bisect_perf_regression._PrepareBisectBranch,
+                                 parent_branch, new_branch)
+
+  def testDeleteBisectBranchIfExists(self):
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    cmds = [
+        (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
+        (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)),
+        (['branch', '-D', new_branch], ('Failed to delete branch', 128))
+        ]
+    self._AssertRunGitExceptions(cmds,
+                                 bisect_perf_regression._PrepareBisectBranch,
+                                 parent_branch, new_branch)
+
+  def testCreatNewBranchFails(self):
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    cmds = [
+        (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
+        (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)),
+        (['branch', '-D', new_branch], ('None', 0)),
+        (['update-index', '--refresh', '-q'], (None, 0)),
+        (['diff-index', 'HEAD'], (None, 0)),
+        (['checkout', '-b', new_branch], ('Failed to create branch', 128))
+        ]
+
+    self._AssertRunGitExceptions(cmds,
+                                 bisect_perf_regression._PrepareBisectBranch,
+                                 parent_branch, new_branch)
+
+  def testSetUpstreamToFails(self):
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    cmds = [
+        (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
+        (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)),
+        (['branch', '-D', new_branch], ('None', 0)),
+        (['update-index', '--refresh', '-q'], (None, 0)),
+        (['diff-index', 'HEAD'], (None, 0)),
+        (['checkout', '-b', new_branch], ('None', 0)),
+        (['branch', '--set-upstream-to', parent_branch],
+         ('Setuptream fails', 1))
+        ]
+    self._AssertRunGitExceptions(cmds,
+                                 bisect_perf_regression._PrepareBisectBranch,
+                                 parent_branch, new_branch)
+  def testBuilderTryJobForException(self):
+    git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
+    bot_name = 'linux_perf_bisect_builder'
+    bisect_job_name = 'testBisectJobname'
+    patch = None
+    patch_content = '/dev/null'
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    try_cmd = [
+        (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
+        (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)),
+        (['branch', '-D', new_branch], ('None', 0)),
+        (['update-index', '--refresh', '-q'], (None, 0)),
+        (['diff-index', 'HEAD'], (None, 0)),
+        (['checkout', '-b', new_branch], ('None', 0)),
+        (['branch', '--set-upstream-to', parent_branch],
+         ('Setuptream fails', 0)),
+        (['try',
+          '-b', bot_name,
+          '-r', git_revision,
+          '-n', bisect_job_name,
+          '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL,
+          '--diff=%s' % patch_content
+          ], (None, 1))
+        ]
+    self._AssertRunGitExceptions(try_cmd,
+                                 bisect_perf_regression._BuilderTryjob,
+                                 git_revision, bot_name, bisect_job_name, patch)
+
+  def testBuilderTryJob(self):
+    git_revision = 'ac4a9f31fe2610bd146857bbd55d7a260003a888'
+    bot_name = 'linux_perf_bisect_builder'
+    bisect_job_name = 'testBisectJobname'
+    patch = None
+    patch_content = '/dev/null'
+    new_branch = bisect_perf_regression.BISECT_TRYJOB_BRANCH
+    parent_branch = bisect_perf_regression.BISECT_MASTER_BRANCH
+    try_cmd = [
+        (['rev-parse', '--abbrev-ref', 'HEAD'], (parent_branch, 0)),
+        (['branch', '--list' ], ('bisect-tryjob\n*master\nsomebranch', 0)),
+        (['branch', '-D', new_branch], ('None', 0)),
+        (['update-index', '--refresh', '-q'], (None, 0)),
+        (['diff-index', 'HEAD'], (None, 0)),
+        (['checkout', '-b', new_branch], ('None', 0)),
+        (['branch', '--set-upstream-to', parent_branch],
+         ('Setuptream fails', 0)),
+        (['try',
+          '-b', bot_name,
+          '-r', git_revision,
+          '-n', bisect_job_name,
+          '--svn_repo=%s' % bisect_perf_regression.SVN_REPO_URL,
+          '--diff=%s' % patch_content
+          ], (None, 0))
+        ]
+    self._SetupRunGitMock(try_cmd)
+    bisect_perf_regression._BuilderTryjob(
+        git_revision, bot_name, bisect_job_name, patch)
+
+
 if __name__ == '__main__':
   unittest.main()

tools/auto_bisect/bisect_utils.py

@@ -11,7 +11,6 @@ annotations for the Buildbot waterfall.
 import errno
 import imp
 import os
-import shutil
 import stat
 import subprocess
 import sys
@@ -83,6 +82,9 @@ REPO_PARAMS = [
     'https://git.chromium.org/external/repo.git'
 ]
 
+# Bisect working directory.
+BISECT_DIR = 'bisect'
+
 
 def OutputAnnotationStepStart(name):
   """Outputs annotation to signal the start of a step to a try bot.
@@ -162,12 +164,12 @@ def _CreateAndChangeToSourceDirectory(working_directory):
   cwd = os.getcwd()
   os.chdir(working_directory)
   try:
-    os.mkdir('bisect')
+    os.mkdir(BISECT_DIR)
   except OSError, e:
     if e.errno != errno.EEXIST:  # EEXIST indicates that it already exists.
       os.chdir(cwd)
       return False
-  os.chdir('bisect')
+  os.chdir(BISECT_DIR)
   return True
 
 
@@ -308,28 +310,6 @@ def OnAccessError(func, path, _):
     raise
 
 
-def RemoveThirdPartyDirectory(dir_name):
-  """Removes third_party directory from the source.
-
-  At some point, some of the third_parties were causing issues to changes in
-  the way they are synced. We remove such folder in order to avoid sync errors
-  while bisecting.
-
-  Returns:
-    True on success, otherwise False.
-  """
-  path_to_dir = os.path.join(os.getcwd(), 'third_party', dir_name)
-  try:
-    if os.path.exists(path_to_dir):
-      shutil.rmtree(path_to_dir, onerror=OnAccessError)
-  except OSError, e:
-    print 'Error #%d while running shutil.rmtree(%s): %s' % (
-        e.errno, path_to_dir, str(e))
-    if e.errno != errno.ENOENT:
-      return False
-  return True
-
-
 def _CleanupPreviousGitRuns():
   """Cleans up any leftover index.lock files after running git."""
   # If a previous run of git crashed, or bot was reset, etc., then we might
@@ -350,7 +330,8 @@ def RunGClientAndSync(cwd=None):
   Returns:
     The return code of the call.
   """
-  params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
+  params = ['sync', '--verbose', '--nohooks', '--reset', '--force',
+            '--delete_unversioned_trees']
   return RunGClient(params, cwd=cwd)
 
 
@@ -368,36 +349,20 @@ def SetupGitDepot(opts, custom_deps):
     otherwise.
   """
   name = 'Setting up Bisection Depot'
-
+  try:
     if opts.output_buildbot_annotations:
       OutputAnnotationStepStart(name)
 
-  passed = False
-
-  if not RunGClientAndCreateConfig(opts, custom_deps):
-    passed_deps_check = True
-    if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
-      cwd = os.getcwd()
-      os.chdir('src')
-      if passed_deps_check:
-        passed_deps_check = RemoveThirdPartyDirectory('libjingle')
-      if passed_deps_check:
-        passed_deps_check = RemoveThirdPartyDirectory('skia')
-      os.chdir(cwd)
+    if RunGClientAndCreateConfig(opts, custom_deps):
+      return False
 
-    if passed_deps_check:
     _CleanupPreviousGitRuns()
-
     RunGClient(['revert'])
-      if not RunGClientAndSync():
-        passed = True
-
+    return not RunGClientAndSync()
+  finally:
     if opts.output_buildbot_annotations:
-    print
       OutputAnnotationStepClosed()
 
-  return passed
-
 
 def CheckIfBisectDepotExists(opts):
   """Checks if the bisect directory already exists.
@@ -408,7 +373,7 @@ def CheckIfBisectDepotExists(opts):
   Returns:
     Returns True if it exists.
   """
-  path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
+  path_to_dir = os.path.join(opts.working_directory, BISECT_DIR, 'src')
   return os.path.exists(path_to_dir)
 
 
@@ -451,6 +416,15 @@ def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
     opts: The options parsed from the command line through parse_args().
     custom_deps: A dictionary of additional dependencies to add to .gclient.
   """
+  if CheckIfBisectDepotExists(opts):
+    path_to_dir = os.path.join(os.path.abspath(opts.working_directory),
+                               BISECT_DIR, 'src')
+    (output, _) = RunGit(['rev-parse', '--is-inside-work-tree'],
+                         cwd=path_to_dir)
+    if output.strip() == 'true':
+      # Checks out the master branch, throws an exception if git command fails.
+      CheckRunGit(['checkout', '-f', 'master'], cwd=path_to_dir)
+
   if not _CreateAndChangeToSourceDirectory(opts.working_directory):
     raise RuntimeError('Could not create bisect directory.')