Add an option to include all subdirs into output

R=stgao@chromium.org, tandrii@chromium.org
BUG=665059

Review-Url: https://codereview.chromium.org/2738123003
Cr-Commit-Position: refs/heads/master@{#456031}

tools/checkteamtags/extract_components.py

@@ -153,13 +153,17 @@ Examples:
                     help='Print complete coverage statistic')
   parser.add_option('-s', '--stat_coverage', type="int",
                     help='Specify directory depth to display coverage stats')
+  parser.add_option('--include-subdirs', action='store_true', default=False,
+                    help='List subdirectories without OWNERS file or component '
+                    'tag as having same component as parent')
   options, args = parser.parse_args(argv[1:])
   if args:
     root = args[0]
   else:
     root = _DEFAULT_SRC_LOCATION
 
-  mappings, warnings, errors, stats = aggregate_components_from_owners(root)
+  mappings, warnings, errors, stats = aggregate_components_from_owners(
+      root, include_subdirs=options.include_subdirs)
   if options.verbose:
     for w in warnings:
       print w

tools/checkteamtags/extract_components_test.py

@@ -2,6 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+from collections import OrderedDict
 import json
 import os
 import sys
@@ -20,8 +21,11 @@ def mock_file_tree(tree):
   os_walk_mocks = []
   file_mocks = {}
   for path in tree:
-    os_walk_mocks.append((path, ('ignored'), ('OWNERS', 'dummy.cc')))
-    file_mocks[os.path.join(path, 'OWNERS')] = tree[path]
+    if tree[path] is not None:
+      os_walk_mocks.append((path, ('ignored'), ('OWNERS', 'dummy.cc')))
+      file_mocks[os.path.join(path, 'OWNERS')] = tree[path]
+    else:
+      os_walk_mocks.append((path, ('ignored'), ('dummy.cc')))
 
   def custom_mock_open(files_data):
     def inner_open(path, mode='r'):
@@ -162,3 +166,35 @@ class ExtractComponentsTest(unittest.TestCase):
     self.assertIn('3 (75.00%) OWNERS files have COMPONENT', output)
     self.assertIn('2 (50.00%) OWNERS files have TEAM and COMPONENT', output)
     self.assertIn('4 OWNERS files at depth 0', output)
+
+  # We use OrderedDict here to guarantee that mocked version of os.walk returns
+  # directories in the specified order (top-down).
+  @mock_file_tree(OrderedDict([
+      ('chromium/src', 'boss@chromium.org\n'),
+      ('chromium/src/dir1', 'dummy@chromium.org\n'
+                           '# TEAM: dummy-team@chromium.org\n'
+                           '# COMPONENT: Dummy>Component'),
+      ('chromium/src/dir2', 'dummy2@chromium.org\n'
+                           '# TEAM: other-dummy-team@chromium.org\n'
+                           '# COMPONENT: Dummy>Component2'),
+      ('chromium/src/dir1/subdir', 'dummy@chromium.org'),
+      ('chromium/src/dir2/subdir', None)]))
+  def testIncludesSubdirectoriesWithNoOwnersFileOrNoComponentTag(self):
+    self.maxDiff = None  # This helps to see assertDictEqual errors in full.
+    saved_output = StringIO()
+    with mock.patch('sys.stdout', saved_output):
+      error_code = extract_components.main(['%prog', '--include-subdirs'])
+    self.assertEqual(0, error_code)
+    result_minus_readme = json.loads(saved_output.getvalue())
+    del result_minus_readme['AAA-README']
+    self.assertDictEqual(result_minus_readme, {
+        u'component-to-team': {
+            u'Dummy>Component2': u'other-dummy-team@chromium.org',
+            u'Dummy>Component': u'dummy-team@chromium.org'
+        },
+        u'dir-to-component': {
+            u'tools/checkteamtags/chromium/src/dir1': u'Dummy>Component',
+            u'tools/checkteamtags/chromium/src/dir1/subdir': u'Dummy>Component',
+            u'tools/checkteamtags/chromium/src/dir2': u'Dummy>Component2',
+            u'tools/checkteamtags/chromium/src/dir2/subdir': u'Dummy>Component2'
+        }})

tools/checkteamtags/owners_file_tags.py

@@ -32,7 +32,7 @@ def parse(filename):
   return team, component
 
 
-def aggregate_components_from_owners(root):
+def aggregate_components_from_owners(root, include_subdirs=False):
   """Traverses the given dir and parse OWNERS files for team and component tags.
 
   Args:
@@ -65,6 +65,7 @@ def aggregate_components_from_owners(root):
   for dirname, _, files in os.walk(root):
     # Proofing against windows casing oddities.
     owners_file_names = [f for f in files if f.upper() == 'OWNERS']
+    rel_dirname = os.path.relpath(dirname, root)
     if owners_file_names:
       file_depth = dirname[len(root) + len(os.path.sep):].count(os.path.sep)
       num_total += 1
@@ -75,13 +76,19 @@ def aggregate_components_from_owners(root):
       if component:
         num_with_component += 1
         num_with_component_by_depth[file_depth] += 1
-        dir_to_component[os.path.relpath(dirname, root)] = component
+        dir_to_component[rel_dirname] = component
         if team:
           num_with_team_component += 1
           num_with_team_component_by_depth[file_depth] += 1
           component_to_team[component].add(team)
       else:
         warnings.append('%s has no COMPONENT tag' % owners_rel_path)
+
+    if include_subdirs and rel_dirname not in dir_to_component:
+      rel_parent_dirname = os.path.relpath(os.path.dirname(dirname), root)
+      if rel_parent_dirname in dir_to_component:
+        dir_to_component[rel_dirname] = dir_to_component[rel_parent_dirname]
+
   mappings = {'component-to-team': component_to_team,
               'dir-to-component': dir_to_component}
   errors = validate_one_team_per_component(mappings)