Commit 47279378 authored by qyearsley's avatar qyearsley Committed by Commit bot

Rename and refactor DownloadCurrentBuild and related functions.

This CL renames a couple functions in order to try to more accurately reflect what they do:
  BuildCurrentRevision -> _ObtainBuild
  DownloadCurrentBuild -> _DownloadAndUnzipBuild

This CL breaks two sub-functions out of DownloadCurrentBuild:
  _WaitForBuildDownload
  _UnzipAndMoveBuildProducts

Note: Later, I plan to replace _GetBuildArchiveForRevision with fetch_build.FetchBuild, and possibly change _UnzipAndMoveBuildProducts so that it doesn't do unzipping, since unzipping is done in fetch_build.FetchBuild. Either that, or make it so that unzipping isn't done in that function.

BUG=

Review URL: https://codereview.chromium.org/664793002

Cr-Commit-Position: refs/heads/master@{#301029}
parent 793ecbcd
...@@ -1073,18 +1073,19 @@ class BisectPerformanceMetrics(object): ...@@ -1073,18 +1073,19 @@ class BisectPerformanceMetrics(object):
return destination_dir return destination_dir
return None return None
def GetBuildArchiveForRevision(self, revision, gs_bucket, target_arch, def _GetBuildArchiveForRevision(self, revision, gs_bucket, target_arch,
patch_sha, out_dir): patch_sha, out_dir):
"""Checks and downloads build archive for a given revision. """Checks and downloads build archive for a given revision.
Checks for build archive with Git hash or SVN revision. If either of the Checks for build archive with Git hash or SVN revision. If either of the
file exists, then downloads the archive file. file exists, then downloads the archive file.
Args: Args:
revision: A Git hash revision. revision: A git commit hash.
gs_bucket: Cloud storage bucket name gs_bucket: Cloud storage bucket name.
target_arch: 32 or 64 bit build target target_arch: Architecture name string, e.g. "ia32" or "x64".
patch: A DEPS patch (used while bisecting 3rd party repositories). patch_sha: A SHA1 hex digest of a DEPS file patch, used while
bisecting 3rd party repositories.
out_dir: Build output directory where downloaded file is stored. out_dir: Build output directory where downloaded file is stored.
Returns: Returns:
...@@ -1104,11 +1105,11 @@ class BisectPerformanceMetrics(object): ...@@ -1104,11 +1105,11 @@ class BisectPerformanceMetrics(object):
return FetchFromCloudStorage(gs_bucket, source_file, out_dir) return FetchFromCloudStorage(gs_bucket, source_file, out_dir)
return downloaded_archive return downloaded_archive
def DownloadCurrentBuild(self, revision, depot, build_type='Release'): def _DownloadAndUnzipBuild(self, revision, depot, build_type='Release'):
"""Downloads the build archive for the given revision. """Downloads the build archive for the given revision.
Args: Args:
revision: The git revision to download or build. revision: The git revision to download.
depot: The name of a dependency repository. Should be in DEPOT_NAMES. depot: The name of a dependency repository. Should be in DEPOT_NAMES.
build_type: Target build type, e.g. Release', 'Debug', 'Release_x64' etc. build_type: Target build type, e.g. Release', 'Debug', 'Release_x64' etc.
...@@ -1129,59 +1130,49 @@ class BisectPerformanceMetrics(object): ...@@ -1129,59 +1130,49 @@ class BisectPerformanceMetrics(object):
# 'DEPS.sha' and add patch_sha evaluated above to it. # 'DEPS.sha' and add patch_sha evaluated above to it.
patch = '%s\n%s' % (patch, DEPS_SHA_PATCH % {'deps_sha': patch_sha}) patch = '%s\n%s' % (patch, DEPS_SHA_PATCH % {'deps_sha': patch_sha})
# Get build output directory.
build_dir = builder.GetBuildOutputDirectory(self.opts, self.src_cwd) build_dir = builder.GetBuildOutputDirectory(self.opts, self.src_cwd)
abs_build_dir = os.path.abspath(build_dir) downloaded_file = self._WaitForBuildDownload(
revision, build_dir, deps_patch=patch, deps_patch_sha=patch_sha)
if not downloaded_file:
return False
return self._UnzipAndMoveBuildProducts(downloaded_file, build_dir,
build_type=build_type)
def _WaitForBuildDownload(self, revision, build_dir, deps_patch=None,
deps_patch_sha=None):
"""Tries to download a zip archive for a build.
This involves seeing whether the archive is already available, and if not,
then requesting a build and waiting before downloading.
fetch_build_func = lambda: self.GetBuildArchiveForRevision( Args:
revision, self.opts.gs_bucket, self.opts.target_arch, revision: A git commit hash.
patch_sha, abs_build_dir) build_dir: The directory to download the build into.
deps_patch: A patch which changes a dependency repository revision in
the DEPS, if applicable.
deps_patch_sha: The SHA1 hex digest of the above patch.
Returns:
File path of the downloaded file if successful, otherwise None.
"""
abs_build_dir = os.path.abspath(build_dir)
fetch_build_func = lambda: self._GetBuildArchiveForRevision(
revision, self.opts.gs_bucket, self.opts.target_arch,
deps_patch_sha, abs_build_dir)
# Downloaded archive file path, downloads build archive for given revision. # Downloaded archive file path, downloads build archive for given revision.
# This will be False if the build isn't yet available.
downloaded_file = fetch_build_func() downloaded_file = fetch_build_func()
# When build archive doesn't exists, post a build request to tryserver # When build archive doesn't exist, post a build request to try server
# and wait for the build to be produced. # and wait for the build to be produced.
if not downloaded_file: if not downloaded_file:
downloaded_file = self._RequestBuildAndWait( downloaded_file = self._RequestBuildAndWait(
revision, fetch_build=fetch_build_func, patch=patch) revision, fetch_build=fetch_build_func, patch=deps_patch)
if not downloaded_file: if not downloaded_file:
return False return None
# Generic name for the archive, created when archive file is extracted.
output_dir = os.path.join(
abs_build_dir, GetZipFileName(target_arch=self.opts.target_arch))
# Unzip build archive directory.
try:
RemoveDirectoryTree(output_dir)
self.BackupOrRestoreOutputDirectory(restore=False)
# Build output directory based on target(e.g. out/Release, out/Debug).
target_build_output_dir = os.path.join(abs_build_dir, build_type)
ExtractZip(downloaded_file, abs_build_dir)
if not os.path.exists(output_dir):
# Due to recipe changes, the builds extract folder contains
# out/Release instead of full-build-<platform>/Release.
if os.path.exists(os.path.join(abs_build_dir, 'out', build_type)):
output_dir = os.path.join(abs_build_dir, 'out', build_type)
else:
raise IOError('Missing extracted folder %s ' % output_dir)
print 'Moving build from %s to %s' % ( return downloaded_file
output_dir, target_build_output_dir)
shutil.move(output_dir, target_build_output_dir)
return True
except Exception as e:
print 'Something went wrong while extracting archive file: %s' % e
self.BackupOrRestoreOutputDirectory(restore=True)
# Cleanup any leftovers from unzipping.
if os.path.exists(output_dir):
RemoveDirectoryTree(output_dir)
finally:
# Delete downloaded archive
if os.path.exists(downloaded_file):
os.remove(downloaded_file)
return False
def _RequestBuildAndWait(self, git_revision, fetch_build, patch=None): def _RequestBuildAndWait(self, git_revision, fetch_build, patch=None):
"""Triggers a try job for a build job. """Triggers a try job for a build job.
...@@ -1252,6 +1243,55 @@ class BisectPerformanceMetrics(object): ...@@ -1252,6 +1243,55 @@ class BisectPerformanceMetrics(object):
return MAX_MAC_BUILD_TIME return MAX_MAC_BUILD_TIME
raise NotImplementedError('Unsupported Platform "%s".' % sys.platform) raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
def _UnzipAndMoveBuildProducts(self, downloaded_file, build_dir,
build_type='Release'):
"""Unzips the build archive and moves it to the build output directory.
The build output directory is whereever the binaries are expected to
be in order to start Chrome and run tests.
Args:
downloaded_file: File path of the downloaded zip file.
build_dir: Directory where the the zip file was downloaded to.
build_type: "Release" or "Debug".
Returns:
True if successful, False otherwise.
"""
abs_build_dir = os.path.abspath(build_dir)
output_dir = os.path.join(
abs_build_dir, GetZipFileName(target_arch=self.opts.target_arch))
try:
RemoveDirectoryTree(output_dir)
self.BackupOrRestoreOutputDirectory(restore=False)
# Build output directory based on target(e.g. out/Release, out/Debug).
target_build_output_dir = os.path.join(abs_build_dir, build_type)
ExtractZip(downloaded_file, abs_build_dir)
if not os.path.exists(output_dir):
# Due to recipe changes, the builds extract folder contains
# out/Release instead of full-build-<platform>/Release.
if os.path.exists(os.path.join(abs_build_dir, 'out', build_type)):
output_dir = os.path.join(abs_build_dir, 'out', build_type)
else:
raise IOError('Missing extracted folder %s ' % output_dir)
print 'Moving build from %s to %s' % (
output_dir, target_build_output_dir)
shutil.move(output_dir, target_build_output_dir)
return True
except Exception as e:
print 'Something went wrong while extracting archive file: %s' % e
self.BackupOrRestoreOutputDirectory(restore=True)
# Cleanup any leftovers from unzipping.
if os.path.exists(output_dir):
RemoveDirectoryTree(output_dir)
finally:
# Delete downloaded archive
if os.path.exists(downloaded_file):
os.remove(downloaded_file)
return False
def IsDownloadable(self, depot): def IsDownloadable(self, depot):
"""Checks if build can be downloaded based on target platform and depot.""" """Checks if build can be downloaded based on target platform and depot."""
if (self.opts.target_platform in ['chromium', 'android'] and if (self.opts.target_platform in ['chromium', 'android'] and
...@@ -1389,11 +1429,16 @@ class BisectPerformanceMetrics(object): ...@@ -1389,11 +1429,16 @@ class BisectPerformanceMetrics(object):
'DEPS checkout Failed for chromium revision : [%s]' % chromium_sha) 'DEPS checkout Failed for chromium revision : [%s]' % chromium_sha)
return (None, None) return (None, None)
def BuildCurrentRevision(self, depot, revision=None): def _ObtainBuild(self, depot, revision=None):
"""Builds chrome and performance_ui_tests on the current revision. """Obtains a build by either downloading or building directly.
Args:
depot: Dependency repository name.
revision: A git commit hash. If None is given, the currently checked-out
revision is built.
Returns: Returns:
True if the build was successful. True for success.
""" """
if self.opts.debug_ignore_build: if self.opts.debug_ignore_build:
return True return True
...@@ -1404,7 +1449,7 @@ class BisectPerformanceMetrics(object): ...@@ -1404,7 +1449,7 @@ class BisectPerformanceMetrics(object):
# Fetch build archive for the given revision from the cloud storage when # Fetch build archive for the given revision from the cloud storage when
# the storage bucket is passed. # the storage bucket is passed.
if self.IsDownloadable(depot) and revision: if self.IsDownloadable(depot) and revision:
build_success = self.DownloadCurrentBuild(revision, depot) build_success = self._DownloadAndUnzipBuild(revision, depot)
else: else:
# These codes are executed when bisect bots builds binaries locally. # These codes are executed when bisect bots builds binaries locally.
build_success = self.builder.Build(depot, self.opts) build_success = self.builder.Build(depot, self.opts)
...@@ -1801,7 +1846,7 @@ class BisectPerformanceMetrics(object): ...@@ -1801,7 +1846,7 @@ class BisectPerformanceMetrics(object):
# Obtain a build for this revision. This may be done by requesting a build # Obtain a build for this revision. This may be done by requesting a build
# from another builder, waiting for it and downloading it. # from another builder, waiting for it and downloading it.
start_build_time = time.time() start_build_time = time.time()
build_success = self.BuildCurrentRevision(depot, revision) build_success = self._ObtainBuild(depot, revision)
if not build_success: if not build_success:
return ('Failed to build revision: [%s]' % str(revision), return ('Failed to build revision: [%s]' % str(revision),
BUILD_RESULT_FAIL) BUILD_RESULT_FAIL)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment