[Findit] Initiate Findit with a unittest skeleton.

This is to create a sub-directory for Findit and add a unittest skeleton with a script to parse DEPS file in chromium.

R=mbarbella@chromium.org

Review URL: https://codereview.chromium.org/391173002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@284775 0039d316-1c4b-4281-b951-d872f2087c98

tools/findit/OWNERS

+stgao@chromium.org

tools/findit/chromium_deps.py

+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import https
+import utils
+
+
+DEPS_FILE_URL = 'https://src.chromium.org/chrome/trunk/src/DEPS?p=%s'
+
+
+class _VarImpl(object):
+  def __init__(self, local_scope):
+    self._local_scope = local_scope
+
+  def Lookup(self, var_name):
+    if var_name in self._local_scope.get('vars', {}):
+      return self._local_scope['vars'][var_name]
+    raise Exception('Var is not defined: %s' % var_name)
+
+
+def _ParseDEPS(content):
+  """Parse the DEPS file of chromium."""
+  local_scope = {}
+  var = _VarImpl(local_scope)
+  global_scope = {
+      'Var': var.Lookup,
+      'deps': {},
+      'deps_os': {},
+      'include_rules': [],
+      'skip_child_includes': [],
+      'hooks': [],
+  }
+  exec(content, global_scope, local_scope)
+
+  local_scope.setdefault('deps', {})
+  local_scope.setdefault('deps_os', {})
+
+  return (local_scope['deps'], local_scope['deps_os'])
+
+
+def _GetComponentName(path):
+  """Return the component name of a path."""
+  host_dirs = [
+      'src/chrome/browser/resources/',
+      'src/chrome/test/data/layout_tests/',
+      'src/media/',
+      'src/sdch/',
+      'src/testing/',
+      'src/third_party/WebKit/',
+      'src/third_party/',
+      'src/tools/',
+      'src/',
+  ]
+  components_renamed = {
+      'webkit': 'blink',
+  }
+
+  for host_dir in host_dirs:
+    if path.startswith(host_dir):
+      path = path[len(host_dir):]
+      name = path.split('/')[0].lower()
+      if name in components_renamed:
+        return components_renamed[name].lower()
+      else:
+        return name.lower()
+
+  # Unknown path, return the whole path as component name.
+  return '_'.join(path.split('/'))
+
+
+def _GetContentOfDEPS(chromium_revision):
+  return https.SendRequest(DEPS_FILE_URL % chromium_revision)
+
+
+def GetChromiumComponents(chromium_revision,
+                          os_platform='unix',
+                          deps_file_downloader=_GetContentOfDEPS):
+  """Return a list of components used by Chrome of the given revision.
+
+  Args:
+    chromium_revision: The revision of the Chrome build.
+    os_platform: The target platform of the Chrome build, eg. win, mac, etc.
+    deps_file_downloader: A function that takes the chromium_revision as input,
+                          and returns the content of the DEPS file. The returned
+                          content is assumed to be trusted input and will be
+                          evaluated as python code.
+  """
+  if os_platform.lower() == 'linux':
+    os_platform = 'unix'
+
+  # Download the content of DEPS file in chromium.
+  deps_content = deps_file_downloader(chromium_revision)
+
+  all_deps = {}
+
+  # Parse the content of DEPS file.
+  deps, deps_os = _ParseDEPS(deps_content)
+  all_deps.update(deps)
+  if os_platform is not None:
+    all_deps.update(deps_os.get(os_platform, {}))
+
+  # Figure out components based on the dependencies.
+  components = {}
+  for component_path in all_deps.keys():
+    name = _GetComponentName(component_path)
+    repository, revision = all_deps[component_path].split('@')
+    is_git_hash = utils.IsGitHash(revision)
+    if repository.startswith('/'):
+      # TODO(stgao): Use git repo after chromium moves to git.
+      repository = 'https://src.chromium.org/chrome%s' % repository
+    if is_git_hash:
+      repository_type = 'git'
+    else:
+      repository_type = 'svn'
+    if not component_path.endswith('/'):
+      component_path += '/'
+    components[component_path] = {
+        'path': component_path,
+        'name': name,
+        'repository': repository,
+        'repository_type': repository_type,
+        'revision': revision
+    }
+
+  # Add chromium as a component.
+  # TODO(stgao): Move to git.
+  components['src/'] = {
+      'path': 'src/',
+      'name': 'chromium',
+      'repository': 'https://src.chromium.org/chrome/trunk',
+      'repository_type': 'svn',
+      'revision': chromium_revision
+  }
+
+  return components
+
+
+def GetChromiumComponentRange(chromium_revision1,
+                              chromium_revision2,
+                              os_platform='unix',
+                              deps_file_downloader=_GetContentOfDEPS):
+  """Return a list of components with their revision ranges.
+
+  Args:
+    chromium_revision1: The revision of a Chrome build.
+    chromium_revision2: The revision of another Chrome build.
+    os_platform: The target platform of the Chrome build, eg. win, mac, etc.
+    deps_file_downloader: A function that takes the chromium_revision as input,
+                          and returns the content of the DEPS file. The returned
+                          content is assumed to be trusted input and will be
+                          evaluated as python code.
+  """
+  # TODO(stgao): support git.
+  chromium_revision1 = int(chromium_revision1)
+  chromium_revision2 = int(chromium_revision2)
+  old_revision = str(min(chromium_revision1, chromium_revision2))
+  new_revision = str(max(chromium_revision1, chromium_revision2))
+
+  old_components = GetChromiumComponents(old_revision, os_platform,
+                                         deps_file_downloader)
+  new_components = GetChromiumComponents(new_revision, os_platform,
+                                         deps_file_downloader)
+
+  components = {}
+  for path in new_components.keys():
+    new_component = new_components[path]
+    old_revision = None
+    if path in old_components:
+      old_component = old_components[path]
+      old_revision = old_component['revision']
+    components[path] = {
+        'path': path,
+        'rolled': new_component['revision'] != old_revision,
+        'name': new_component['name'],
+        'old_revision': old_revision,
+        'new_revision': new_component['revision'],
+        'repository': new_component['repository'],
+        'repository_type': new_component['repository_type']
+    }
+
+  return components
+
+
+if __name__ == '__main__':
+  import json
+  print json.dumps(GetChromiumComponents(284750), sort_keys=True, indent=2)

tools/findit/chromium_deps_unittest.py

+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+import chromium_deps
+
+
+class ChromiumDEPSTest(unittest.TestCase):
+  DEPS_TEMPLATE = """
+vars = {
+  "googlecode_url": "http://%%s.googlecode.com/svn",
+  "webkit_trunk": "http://src.chromium.org/blink/trunk",
+  "webkit_revision": "%s",
+  "chromium_git": "https://chromium.googlesource.com",
+}
+
+deps = {
+  "src/breakpad/src":
+    (Var("googlecode_url") %% "google-breakpad") + "/trunk/src@%s",
+
+  "src/third_party/WebKit":
+    Var("webkit_trunk") + "@" + Var("webkit_revision"),
+}
+
+deps_os = {
+  "unix": {
+    "src/third_party/liblouis/src":
+      Var("chromium_git") +
+      "/external/liblouis.git@%s",
+  }
+}
+"""
+
+  def __init__(self, *args, **kwargs):
+    super(ChromiumDEPSTest, self).__init__(*args, **kwargs)
+
+  def testGetChromiumComponents(self):
+    chromium_revision = '283296'
+    webkit_revision = '178200'
+    breakpad_revision = '1345'
+    liblouis_commit_hashcode = '3c2daee56250162e5a75830871601d74328d39f5'
+
+    def _GetContentOfDEPS(chromium_revision_tmp):
+      self.assertEqual(chromium_revision_tmp, chromium_revision)
+      return self.DEPS_TEMPLATE % (webkit_revision, breakpad_revision,
+                                   liblouis_commit_hashcode)
+
+    expected_results = {
+        'src/breakpad/src/': {
+            'path': 'src/breakpad/src/',
+            'repository_type': 'svn',
+            'name': 'breakpad',
+            'repository': 'http://google-breakpad.googlecode.com/svn/trunk/src',
+            'revision': breakpad_revision
+        },
+        'src/third_party/liblouis/src/': {
+            'path': 'src/third_party/liblouis/src/',
+            'repository_type': 'git',
+            'name': 'liblouis',
+            'repository':
+                 'https://chromium.googlesource.com/external/liblouis.git',
+            'revision': liblouis_commit_hashcode
+        },
+        'src/': {
+            'path': 'src/',
+            'repository_type': 'svn',
+            'name': 'chromium',
+            'repository': 'https://src.chromium.org/chrome/trunk',
+            'revision': chromium_revision
+        },
+        'src/third_party/WebKit/': {
+            'path': 'src/third_party/WebKit/',
+            'repository_type': 'svn',
+            'name': 'blink',
+            'repository': 'http://src.chromium.org/blink/trunk',
+            'revision': webkit_revision
+        }
+    }
+
+    components = chromium_deps.GetChromiumComponents(
+        chromium_revision, deps_file_downloader=_GetContentOfDEPS)
+    self.assertEqual(expected_results, components)
+
+  def testGetChromiumComponentRange(self):
+    chromium_revision1 = '283296'
+    webkit_revision1 = '178200'
+    breakpad_revision1 = '1345'
+    liblouis_commit_hashcode1 = '3c2daee56250162e5a75830871601d74328d39f5'
+
+    chromium_revision2 = '283200'
+    webkit_revision2 = '178084'
+    breakpad_revision2 = '1345'
+    liblouis_commit_hashcode2 = '3c2daee56250162e5a75830871601d74328d39f5'
+
+    def _GetContentOfDEPS(chromium_revision):
+      chromium_revision = str(chromium_revision)
+      if chromium_revision == chromium_revision1:
+        return self.DEPS_TEMPLATE % (webkit_revision1, breakpad_revision1,
+                                     liblouis_commit_hashcode1)
+      else:
+        self.assertEqual(chromium_revision2, chromium_revision)
+        return self.DEPS_TEMPLATE % (webkit_revision2, breakpad_revision2,
+                                     liblouis_commit_hashcode2)
+
+    expected_results = {
+        'src/breakpad/src/': {
+            'old_revision': breakpad_revision2,
+            'name': 'breakpad',
+            'repository': 'http://google-breakpad.googlecode.com/svn/trunk/src',
+            'rolled': False,
+            'new_revision': breakpad_revision1,
+            'path': 'src/breakpad/src/',
+            'repository_type': 'svn'
+        },
+        'src/third_party/liblouis/src/': {
+            'old_revision': liblouis_commit_hashcode2,
+            'name': 'liblouis',
+            'repository':
+                'https://chromium.googlesource.com/external/liblouis.git',
+            'rolled': False,
+            'new_revision': liblouis_commit_hashcode1,
+            'path': 'src/third_party/liblouis/src/',
+            'repository_type': 'git'
+        },
+        'src/': {
+            'old_revision': chromium_revision2,
+            'name': 'chromium',
+            'repository': 'https://src.chromium.org/chrome/trunk',
+            'rolled': True,
+            'new_revision': chromium_revision1,
+            'path': 'src/',
+            'repository_type': 'svn'
+        },
+        'src/third_party/WebKit/': {
+            'old_revision': webkit_revision2,
+            'name': 'blink',
+            'repository': 'http://src.chromium.org/blink/trunk',
+            'rolled': True,
+            'new_revision': webkit_revision1,
+            'path': 'src/third_party/WebKit/',
+            'repository_type': 'svn'
+        }
+    }
+
+    components = chromium_deps.GetChromiumComponentRange(
+        chromium_revision1, chromium_revision2,
+        deps_file_downloader=_GetContentOfDEPS)
+    self.assertEqual(expected_results, components)

tools/findit/https.py

+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+Provides a utility function for https connections with certificate verification.
+
+The verification is based on http://tools.ietf.org/html/rfc6125#section-6.4.3
+and the code is from Lib/ssl.py in python3:
+  http://hg.python.org/cpython/file/4dac45f88d45/Lib/ssl.py
+
+One use case is to download Chromium DEPS file in a secure way:
+  https://src.chromium.org/chrome/trunk/src/DEPS
+
+Notice: python 2.7 or newer is required.
+"""
+
+import httplib
+import os
+import re
+import socket
+import ssl
+import urllib2
+
+
+_SCRIPT_DIR = os.path.dirname(__file__)
+_TRUSTED_ROOT_CERTS = os.path.join(_SCRIPT_DIR, 'cacert.pem')
+
+
+class CertificateError(ValueError):
+  pass
+
+
+def _DNSNameMatch(dn, hostname, max_wildcards=1):
+  """Matching according to RFC 6125, section 6.4.3
+
+  http://tools.ietf.org/html/rfc6125#section-6.4.3
+  """
+  pats = []
+  if not dn:
+    return False
+
+  parts = dn.split(r'.')
+  leftmost = parts[0]
+  remainder = parts[1:]
+
+  wildcards = leftmost.count('*')
+  if wildcards > max_wildcards:
+    # Issue #17980: avoid denials of service by refusing more
+    # than one wildcard per fragment.  A survery of established
+    # policy among SSL implementations showed it to be a
+    # reasonable choice.
+    raise CertificateError(
+        'too many wildcards in certificate DNS name: ' + repr(dn))
+
+  # speed up common case w/o wildcards
+  if not wildcards:
+    return dn.lower() == hostname.lower()
+
+  # RFC 6125, section 6.4.3, subitem 1.
+  # The client SHOULD NOT attempt to match a presented identifier in which
+  # the wildcard character comprises a label other than the left-most label.
+  if leftmost == '*':
+    # When '*' is a fragment by itself, it matches a non-empty dotless
+    # fragment.
+    pats.append('[^.]+')
+  elif leftmost.startswith('xn--') or hostname.startswith('xn--'):
+    # RFC 6125, section 6.4.3, subitem 3.
+    # The client SHOULD NOT attempt to match a presented identifier
+    # where the wildcard character is embedded within an A-label or
+    # U-label of an internationalized domain name.
+    pats.append(re.escape(leftmost))
+  else:
+    # Otherwise, '*' matches any dotless string, e.g. www*
+    pats.append(re.escape(leftmost).replace(r'\*', '[^.]*'))
+
+  # add the remaining fragments, ignore any wildcards
+  for frag in remainder:
+    pats.append(re.escape(frag))
+
+  pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+  return pat.match(hostname)
+
+
+def _MatchHostname(cert, hostname):
+  """Verify that *cert* (in decoded format as returned by
+  SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 and RFC 6125
+  rules are followed, but IP addresses are not accepted for *hostname*.
+
+  CertificateError is raised on failure. On success, the function
+  returns nothing.
+  """
+  if not cert:
+    raise ValueError('empty or no certificate, match_hostname needs a '
+                     'SSL socket or SSL context with either '
+                     'CERT_OPTIONAL or CERT_REQUIRED')
+  dnsnames = []
+  san = cert.get('subjectAltName', ())
+  for key, value in san:
+    if key == 'DNS':
+      if _DNSNameMatch(value, hostname):
+        return
+      dnsnames.append(value)
+  if not dnsnames:
+    # The subject is only checked when there is no dNSName entry
+    # in subjectAltName
+    for sub in cert.get('subject', ()):
+      for key, value in sub:
+        # XXX according to RFC 2818, the most specific Common Name
+        # must be used.
+        if key == 'commonName':
+          if _DNSNameMatch(value, hostname):
+            return
+          dnsnames.append(value)
+  if len(dnsnames) > 1:
+    raise CertificateError('hostname %r doesn\'t match either of %s'
+                           % (hostname, ', '.join(map(repr, dnsnames))))
+  elif len(dnsnames) == 1:
+    raise CertificateError('hostname %r doesn\'t match %r'
+                           % (hostname, dnsnames[0]))
+  else:
+    raise CertificateError('no appropriate commonName or '
+                           'subjectAltName fields were found')
+
+
+class HTTPSConnection(httplib.HTTPSConnection):
+
+  def __init__(self, host, root_certs=_TRUSTED_ROOT_CERTS, **kwargs):
+    self.root_certs = root_certs
+    httplib.HTTPSConnection.__init__(self, host, **kwargs)
+
+  def connect(self):
+    # Overrides for certificate verification.
+    args = [(self.host, self.port), self.timeout,]
+    if self.source_address:
+      args.append(self.source_address)
+    sock = socket.create_connection(*args)
+
+    if self._tunnel_host:
+      self.sock = sock
+      self._tunnel()
+
+    # Wrap the socket for verification with the root certs.
+    kwargs = {}
+    if self.root_certs is not None:
+      kwargs.update(cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.root_certs)
+    self.sock = ssl.wrap_socket(sock, **kwargs)
+
+    # Check hostname.
+    try:
+      _MatchHostname(self.sock.getpeercert(), self.host)
+    except CertificateError:
+      self.sock.shutdown(socket.SHUT_RDWR)
+      self.sock.close()
+      raise
+
+
+class HTTPSHandler(urllib2.HTTPSHandler):
+
+  def __init__(self, root_certs=_TRUSTED_ROOT_CERTS):
+    urllib2.HTTPSHandler.__init__(self)
+    self.root_certs = root_certs
+
+  def https_open(self, req):
+    # Pass a reference to the function below so that verification against
+    # trusted root certs could be injected.
+    return self.do_open(self.GetConnection, req)
+
+  def GetConnection(self, host, **kwargs):
+    params = dict(root_certs=self.root_certs)
+    params.update(kwargs)
+    return HTTPSConnection(host, **params)
+
+
+def SendRequest(https_url):
+  """Send request to the given https url, and return the server response.
+
+  Args:
+    https_url: The https url to send request to.
+
+  Returns:
+    A string that is the response from the server.
+
+  Raises:
+    ValueError: Unexpected value is received during certificate verification.
+    CertificateError: Certificate verification fails.
+  """
+  if not https_url or not https_url.startswith('https://'):
+    raise ValueError('Not a https request for url %s.' % str(https_url))
+
+  url_opener = urllib2.build_opener(HTTPSHandler)
+  return url_opener.open(https_url).read()
+
+
+if __name__ == '__main__':
+  print SendRequest('https://src.chromium.org/chrome/trunk/src/DEPS')

tools/findit/run_all_tests.py

+#!/usr/bin/env python
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import sys
+import unittest
+
+from chromium_deps_unittest import ChromiumDEPSTest
+
+
+if __name__ == '__main__':
+   all_tests_suite = unittest.defaultTestLoader.loadTestsFromModule(
+             sys.modules[__name__])
+   tests = unittest.TestSuite(all_tests_suite)
+   result = unittest.TextTestRunner(stream=sys.stdout, verbosity=2).run(tests)
+   sys.exit(len(result.failures) + len(result.errors))

tools/findit/utils.py

+# Copyright (c) 2011 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import re
+import sys
+
+
+GIT_HASH_PATTERN = re.compile(r'^[0-9a-fA-F]{40}$')
+
+
+def GetOSName(platform_name=sys.platform):
+  if platform_name == 'cygwin' or platform_name.startswith('win'):
+    return 'win'
+  elif platform_name.startswith('linux'):
+    return 'unix'
+  elif platform_name.startswith('darwin'):
+    return 'mac'
+  else:
+    return platform_name
+
+
+def IsGitHash(revision):
+  return GIT_HASH_PATTERN.match(str(revision))