Remove find_dependencies script

BUG=chromium:728177

Review-Url: https://codereview.chromium.org/2932213002
Cr-Commit-Position: refs/heads/master@{#478658}

tools/perf/core/bootstrap.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Bootstrap Chrome Telemetry by downloading all its files from SVN servers.
-
-Requires a DEPS file to specify which directories on which SVN servers
-are required to run Telemetry. Format of that DEPS file is a subset of the
-normal DEPS file format[1]; currently only only the "deps" dictionary is
-supported and nothing else.
-
-Fetches all files in the specified directories using WebDAV (SVN is WebDAV under
-the hood).
-
-[1] http://dev.chromium.org/developers/how-tos/depottools#TOC-DEPS-file
-"""
-
-import imp
-import logging
-import os
-import urllib
-import urlparse
-
-# Dummy module for DAVclient.
-davclient = None
-
-
-# TODO(eakuefner): Switch this link to tools/perf version after verifying.
-# Link to file containing the 'davclient' WebDAV client library.
-_DAVCLIENT_URL = ('https://src.chromium.org/chrome/trunk/src/tools/'
-                  'telemetry/third_party/davclient/davclient.py')
-
-
-def _DownloadAndImportDAVClientModule():
-  """Dynamically import davclient helper library."""
-  global davclient
-  davclient_src = urllib.urlopen(_DAVCLIENT_URL).read()
-  davclient = imp.new_module('davclient')
-  exec davclient_src in davclient.__dict__  # pylint: disable=exec-used
-
-
-class DAVClientWrapper(object):
-  """Knows how to retrieve subdirectories and files from WebDAV/SVN servers."""
-
-  def __init__(self, root_url):
-    """Initialize SVN server root_url, save files to local dest_dir.
-
-    Args:
-      root_url: string url of SVN/WebDAV server
-    """
-    self.root_url = root_url
-    self.client = davclient.DAVClient(root_url)
-
-  @staticmethod
-  def __norm_path_keys(dict_with_path_keys):
-    """Returns a dictionary with os.path.normpath called on every key."""
-    return dict((os.path.normpath(k), v) for (k, v) in
-                dict_with_path_keys.items())
-
-  def GetDirList(self, path):
-    """Returns string names of all files and subdirs of path on the server."""
-    props = self.__norm_path_keys(self.client.propfind(path, depth=1))
-    # remove this path
-    del props[os.path.normpath(path)]
-    return [os.path.basename(p) for p in props.keys()]
-
-  def IsFile(self, path):
-    """Returns True if the path is a file on the server, False if directory."""
-    props = self.__norm_path_keys(self.client.propfind(path, depth=1))
-    return props[os.path.normpath(path)]['resourcetype'] is None
-
-  def Traverse(self, src_path, dst_path):
-    """Walks the directory hierarchy pointed to by src_path download all files.
-
-    Recursively walks src_path and saves all files and subfolders into
-    dst_path.
-
-    Args:
-      src_path: string path on SVN server to save (absolute path on server).
-      dest_path: string local path (relative or absolute) to save to.
-    """
-    if self.IsFile(src_path):
-      if not os.path.exists(os.path.dirname(dst_path)):
-        logging.info('Creating %s', os.path.dirname(dst_path))
-        os.makedirs(os.path.dirname(dst_path))
-      if os.path.isfile(dst_path):
-        logging.info('Skipping %s', dst_path)
-      else:
-        logging.info('Saving %s to %s', self.root_url + src_path, dst_path)
-        urllib.urlretrieve(self.root_url + src_path, dst_path)
-      return
-    else:
-      for subdir in self.GetDirList(src_path):
-        self.Traverse(os.path.join(src_path, subdir),
-                      os.path.join(dst_path, subdir))
-
-
-def ListAllDepsPaths(deps_file):
-  """Recursively returns a list of all paths indicated in this deps file.
-
-  Note that this discards information about where path dependencies come from,
-  so this is only useful in the context of a Chromium source checkout that has
-  already fetched all dependencies.
-
-  Args:
-    deps_file: File containing deps information to be evaluated, in the
-               format given in the header of this file.
-  Returns:
-    A list of string paths starting under src that are required by the
-    given deps file, and all of its sub-dependencies. This amounts to
-    the keys of the 'deps' dictionary.
-  """
-  deps = {}
-  deps_includes = {}
-
-  chrome_root = os.path.dirname(__file__)
-  while os.path.basename(chrome_root) != 'src':
-    chrome_root = os.path.abspath(os.path.join(chrome_root, '..'))
-
-  exec open(deps_file).read()  # pylint: disable=exec-used
-
-  deps_paths = deps.keys()
-
-  for path in deps_includes.keys():
-    # Need to localize the paths.
-    path = os.path.join(chrome_root, '..', path)
-    deps_paths += ListAllDepsPaths(path)
-
-  return deps_paths
-
-
-def DownloadDeps(destination_dir, url):
-  """Saves all the dependencies in deps_path.
-
-  Opens and reads url, assuming the contents are in the simple DEPS-like file
-  format specified in the header of this file, then download all
-  files/directories listed to the destination_dir.
-
-  Args:
-    destination_dir: String path to directory to download files into.
-    url: URL containing deps information to be evaluated.
-  """
-  logging.warning('Downloading deps from %s...', url)
-  # TODO(wiltzius): Add a parameter for which revision to pull.
-  _DownloadAndImportDAVClientModule()
-
-  deps = {}
-  deps_includes = {}
-
-  exec urllib.urlopen(url).read()  # pylint: disable=exec-used
-
-  for dst_path, src_path in deps.iteritems():
-    full_dst_path = os.path.join(destination_dir, dst_path)
-    parsed_url = urlparse.urlparse(src_path)
-    root_url = parsed_url.scheme + '://' + parsed_url.netloc
-
-    dav_client = DAVClientWrapper(root_url)
-    dav_client.Traverse(parsed_url.path, full_dst_path)
-
-  for url in deps_includes.values():
-    DownloadDeps(destination_dir, url)

tools/perf/core/find_dependencies.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import fnmatch
-import imp
-import logging
-import optparse
-import os
-import sys
-import zipfile
-
-from telemetry import benchmark
-from telemetry.core import discover
-from telemetry.internal.util import command_line
-from telemetry.internal.util import path
-from telemetry.internal.util import path_set
-
-try:
-  from modulegraph import modulegraph  # pylint: disable=import-error
-except ImportError as err:
-  modulegraph = None
-  import_error = err
-
-from core import bootstrap
-from core import path_util
-
-DEPS_FILE = 'bootstrap_deps'
-
-
-def FindBootstrapDependencies(base_dir):
-  deps_file = os.path.join(base_dir, DEPS_FILE)
-  if not os.path.exists(deps_file):
-    return []
-  deps_paths = bootstrap.ListAllDepsPaths(deps_file)
-  return set(os.path.realpath(os.path.join(
-      path_util.GetChromiumSrcDir(), '..', deps_path))
-             for deps_path in deps_paths)
-
-
-def FindPythonDependencies(module_path):
-  logging.info('Finding Python dependencies of %s', module_path)
-  if modulegraph is None:
-    raise import_error
-
-  sys_path = sys.path
-  sys.path = list(sys_path)
-  try:
-    # Load the module to inherit its sys.path modifications.
-    sys.path.insert(0, os.path.abspath(os.path.dirname(module_path)))
-    imp.load_source(
-        os.path.splitext(os.path.basename(module_path))[0], module_path)
-
-    # Analyze the module for its imports.
-    graph = modulegraph.ModuleGraph()
-    graph.run_script(module_path)
-
-    # Filter for only imports in Chromium.
-    for node in graph.nodes():
-      if not node.filename:
-        continue
-      module_path = os.path.realpath(node.filename)
-
-      _, incoming_edges = graph.get_edges(node)
-      message = 'Discovered %s (Imported by: %s)' % (
-          node.filename, ', '.join(
-              d.filename for d in incoming_edges
-              if d is not None and d.filename is not None))
-      logging.info(message)
-
-      # This check is done after the logging/printing above to make sure that
-      # we also print out the dependency edges that include python packages
-      # that are not in chromium.
-      if not path.IsSubpath(module_path, path_util.GetChromiumSrcDir()):
-        continue
-
-      yield module_path
-      if node.packagepath is not None:
-        for p in node.packagepath:
-          yield p
-
-  finally:
-    sys.path = sys_path
-
-
-def FindPageSetDependencies(base_dir):
-  logging.info('Finding page sets in %s', base_dir)
-
-  # Add base_dir to path so our imports relative to base_dir will work.
-  sys.path.append(base_dir)
-  tests = discover.DiscoverClasses(base_dir, base_dir, benchmark.Benchmark,
-                                   index_by_class_name=True)
-
-  for test_class in tests.itervalues():
-    test_obj = test_class()
-
-    # Ensure the test's default options are set if needed.
-    parser = optparse.OptionParser()
-    test_obj.AddCommandLineArgs(parser, None)
-    options = optparse.Values()
-    for k, v in parser.get_default_values().__dict__.iteritems():
-      options.ensure_value(k, v)
-
-    # Page set paths are relative to their runner script, not relative to us.
-    path.GetBaseDir = lambda: base_dir
-    # TODO: Loading the page set will automatically download its Cloud Storage
-    # deps. This is really expensive, and we don't want to do this by default.
-    story_set = test_obj.CreateStorySet(options)
-
-    # Add all of its serving_dirs as dependencies.
-    for serving_dir in story_set.serving_dirs:
-      yield serving_dir
-
-
-def FindExcludedFiles(files, options):
-  # Define some filters for files.
-  def IsHidden(path_string):
-    for pathname_component in path_string.split(os.sep):
-      if pathname_component.startswith('.'):
-        return True
-    return False
-
-  def IsPyc(path_string):
-    return os.path.splitext(path_string)[1] == '.pyc'
-
-  def IsInCloudStorage(path_string):
-    return os.path.exists(path_string + '.sha1')
-
-  def MatchesExcludeOptions(path_string):
-    for pattern in options.exclude:
-      if (fnmatch.fnmatch(path_string, pattern) or
-          fnmatch.fnmatch(os.path.basename(path_string), pattern)):
-        return True
-    return False
-
-  # Collect filters we're going to use to exclude files.
-  exclude_conditions = [
-      IsHidden,
-      IsPyc,
-      IsInCloudStorage,
-      MatchesExcludeOptions,
-  ]
-
-  # Check all the files against the filters.
-  for file_path in files:
-    if any(condition(file_path) for condition in exclude_conditions):
-      yield file_path
-
-
-def FindDependencies(target_paths, options):
-  # Verify arguments.
-  for target_path in target_paths:
-    if not os.path.exists(target_path):
-      raise ValueError('Path does not exist: %s' % target_path)
-
-  dependencies = path_set.PathSet()
-
-  # Including Telemetry's major entry points will (hopefully) include Telemetry
-  # and all its dependencies. If the user doesn't pass any arguments, we just
-  # have Telemetry.
-  dependencies |= FindPythonDependencies(os.path.realpath(
-      os.path.join(path_util.GetTelemetryDir(),
-                   'telemetry', 'benchmark_runner.py')))
-  dependencies |= FindPythonDependencies(os.path.realpath(
-      os.path.join(path_util.GetTelemetryDir(),
-                   'telemetry', 'testing', 'run_tests.py')))
-
-  # Add dependencies.
-  for target_path in target_paths:
-    base_dir = os.path.dirname(os.path.realpath(target_path))
-
-    dependencies.add(base_dir)
-    dependencies |= FindBootstrapDependencies(base_dir)
-    dependencies |= FindPythonDependencies(target_path)
-    if options.include_page_set_data:
-      dependencies |= FindPageSetDependencies(base_dir)
-
-  # Remove excluded files.
-  dependencies -= FindExcludedFiles(set(dependencies), options)
-
-  return dependencies
-
-
-def ZipDependencies(target_paths, dependencies, options):
-  base_dir = os.path.dirname(os.path.realpath(path_util.GetChromiumSrcDir()))
-
-  with zipfile.ZipFile(options.zip, 'w', zipfile.ZIP_DEFLATED) as zip_file:
-    # Add dependencies to archive.
-    for dependency_path in dependencies:
-      path_in_archive = os.path.join(
-          'telemetry', os.path.relpath(dependency_path, base_dir))
-      zip_file.write(dependency_path, path_in_archive)
-
-    # Add symlinks to executable paths, for ease of use.
-    for target_path in target_paths:
-      link_info = zipfile.ZipInfo(
-          os.path.join('telemetry', os.path.basename(target_path)))
-      link_info.create_system = 3  # Unix attributes.
-      # 010 is regular file, 0111 is the permission bits rwxrwxrwx.
-      link_info.external_attr = 0100777 << 16  # Octal.
-
-      relative_path = os.path.relpath(target_path, base_dir)
-      link_script = (
-          '#!/usr/bin/env python\n\n'
-          'import os\n'
-          'import sys\n\n\n'
-          'script = os.path.join(os.path.dirname(__file__), \'%s\')\n'
-          'os.execv(sys.executable, [sys.executable, script] + sys.argv[1:])'
-          % relative_path)
-
-      zip_file.writestr(link_info, link_script)
-
-
-class FindDependenciesCommand(command_line.OptparseCommand):
-  """Prints all dependencies"""
-
-  @classmethod
-  def AddCommandLineArgs(cls, parser, _):
-    parser.add_option(
-        '-v', '--verbose', action='count', dest='verbosity',
-        help='Increase verbosity level (repeat as needed).')
-
-    parser.add_option(
-        '-p', '--include-page-set-data', action='store_true', default=False,
-        help='Scan tests for page set data and include them.')
-
-    parser.add_option(
-        '-e', '--exclude', action='append', default=[],
-        help='Exclude paths matching EXCLUDE. Can be used multiple times.')
-
-    parser.add_option(
-        '-z', '--zip',
-        help='Store files in a zip archive at ZIP.')
-
-  @classmethod
-  def ProcessCommandLineArgs(cls, parser, args, _):
-    if args.verbosity >= 2:
-      logging.getLogger().setLevel(logging.DEBUG)
-    elif args.verbosity:
-      logging.getLogger().setLevel(logging.INFO)
-    else:
-      logging.getLogger().setLevel(logging.WARNING)
-
-  def Run(self, args):
-    target_paths = args.positional_args
-    dependencies = FindDependencies(target_paths, args)
-    if args.zip:
-      ZipDependencies(target_paths, dependencies, args)
-      print 'Zip archive written to %s.' % args.zip
-    else:
-      print '\n'.join(sorted(dependencies))
-    return 0

tools/perf/core/find_dependencies_unittest.py

-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import os
-import unittest
-
-from telemetry.core import util
-
-from core import find_dependencies
-
-
-class FindDependenciesTest(unittest.TestCase):
-
-  def testFindPythonDependencies(self):
-    try:
-      dog_object_path = os.path.join(
-          util.GetUnittestDataDir(),
-          'dependency_test_dir', 'dog', 'dog', 'dog_object.py')
-      cat_module_path = os.path.join(
-          util.GetUnittestDataDir(),
-          'dependency_test_dir', 'other_animals', 'cat', 'cat')
-      cat_module_init_path = os.path.join(cat_module_path, '__init__.py')
-      cat_object_path = os.path.join(cat_module_path, 'cat_object.py')
-      self.assertEquals(
-          set(p for p in
-              find_dependencies.FindPythonDependencies(dog_object_path)),
-          {dog_object_path, cat_module_path, cat_module_init_path,
-           cat_object_path})
-    except ImportError:  # crbug.com/559527
-      pass
-
-  def testFindPythonDependenciesWithNestedImport(self):
-    try:
-      moose_module_path = os.path.join(
-          util.GetUnittestDataDir(),
-          'dependency_test_dir', 'other_animals', 'moose', 'moose')
-      moose_object_path = os.path.join(moose_module_path, 'moose_object.py')
-      horn_module_path = os.path.join(moose_module_path, 'horn')
-      horn_module_init_path = os.path.join(horn_module_path, '__init__.py')
-      horn_object_path = os.path.join(horn_module_path, 'horn_object.py')
-      self.assertEquals(
-          set(p for p in
-              find_dependencies.FindPythonDependencies(moose_object_path)),
-          {moose_object_path,
-           horn_module_path, horn_module_init_path, horn_object_path})
-    except ImportError:   # crbug.com/559527
-      pass

tools/perf/find_dependencies

-#!/usr/bin/env python
-# Copyright 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import sys
-
-from core import find_dependencies
-
-
-if __name__ == '__main__':
-  find_dependencies.FindDependenciesCommand.main()

tools/perf/third_party/__init__.py

-# Copyright 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.

tools/perf/third_party/davclient/README.chromium

-Name: Davclient 
-Short Name: davclient
-URL: http://svn.osafoundation.org/tools/davclient/ 
-Version: 0.2.1
-Date: Undated, retrieved Dec 14, 2012
-Revision: r1008 
-License: Apache 2.0
-License File: NOT_SHIPPED
-Security Critical: no
-
-Description:
-A simple Python WebDAV client. Used for the Telemetry bootstrap to easily
-fetch files from SVN servers.
-
-Local Modifications:
-None. However packaging and installation code from original repository have not
-been copied.

tools/perf/third_party/davclient/__init__.py

-# Copyright 2015 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.