generate_buildbot_json.py: Better handle printed lines

This CL changes the tests for generate_buildbot_json.py to capture all
printed lines. It requires tests to assert about the contents of these
printed lines, and then clear them.

It also adds a --verbose flag to the main script, which is passed when
checking the configs.

Also fixes a test for swarming mixins and how they interact with certain
classes of tests.

Bug: 886993
Change-Id: I047d74689d72f35bc7d19c8da338c070a8a5d9a5
Reviewed-on: https://chromium-review.googlesource.com/1237239
Commit-Queue: Stephen Martinis <martiniss@chromium.org>
Reviewed-by: Kenneth Russell <kbr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#593015}

testing/buildbot/PRESUBMIT.py

@@ -13,7 +13,8 @@ def CommonChecks(input_api, output_api):
   commands = [
     input_api.Command(
       name='generate_buildbot_json', cmd=[
-        input_api.python_executable, 'generate_buildbot_json.py', '--check'],
+        input_api.python_executable, 'generate_buildbot_json.py', '--check',
+        '--verbose'],
       kwargs={}, message=output_api.PresubmitError),
 
     input_api.Command(

testing/buildbot/generate_buildbot_json.py

@@ -197,6 +197,10 @@ class BBJSONGenerator(object):
   def generate_abs_file_path(self, relative_path):
     return os.path.join(self.this_dir, relative_path) # pragma: no cover
 
+  def print_line(self, line):
+    # Exists so that tests can mock
+    print line # pragma: no cover
+
   def read_file(self, relative_path):
     with open(self.generate_abs_file_path(
         relative_path)) as fp: # pragma: no cover
@@ -984,7 +988,7 @@ class BBJSONGenerator(object):
           for line in difflib.unified_diff(
               sorted(keys),
               keys):
-            print line
+            self.print_line(line)
 
     if bad_files:
       raise BBGenErr(
@@ -1004,20 +1008,21 @@ class BBJSONGenerator(object):
       if expected != current:
         ungenerated_waterfalls.add(waterfall['name'])
         if verbose: # pragma: no cover
-          print ('Waterfall ' +  waterfall['name'] +
+          self.print_line('Waterfall ' +  waterfall['name'] +
                  ' did not have the following expected '
                  'contents:')
           for line in difflib.unified_diff(
               expected.splitlines(),
-              current.splitlines()):
-            print line
+              current.splitlines(),
+              fromfile='expected', tofile='current'):
+            self.print_line(line)
     if ungenerated_waterfalls:
       raise BBGenErr('The following waterfalls have not been properly '
                      'autogenerated by generate_buildbot_json.py: ' +
                      str(ungenerated_waterfalls))
 
   def check_consistency(self, verbose=False):
-    self.check_input_file_consistency() # pragma: no cover
+    self.check_input_file_consistency(verbose) # pragma: no cover
     self.check_output_file_consistency(verbose) # pragma: no cover
 
   def parse_args(self, argv): # pragma: no cover
@@ -1030,6 +1035,9 @@ class BBJSONGenerator(object):
       '-n', '--new-files', action='store_true', help=
       'Write output files as .new.json. Useful during development so old and '
       'new files can be looked at side-by-side.')
+    parser.add_argument(
+      '-v', '--verbose', action='store_true', help=
+      'Increases verbosity. Affects consistency checks.')
     parser.add_argument(
       'waterfall_filters', metavar='waterfalls', type=str, nargs='*',
       help='Optional list of waterfalls to generate.')
@@ -1041,7 +1049,7 @@ class BBJSONGenerator(object):
   def main(self, argv): # pragma: no cover
     self.parse_args(argv)
     if self.args.check:
-      self.check_consistency()
+      self.check_consistency(verbose=self.args.verbose)
     else:
       self.generate_waterfalls()
     return 0