Add autogenerated CQ documentation with presubmit

The presubmit check ensures that the builders are sorted, first by group,
then by name. It also ensures that the file generated by the script is up
to date.

Bug: 902507

Change-Id: Iaaf7048b6b6edcc99befff7c3e8602adfd2c5681
Reviewed-on: https://chromium-review.googlesource.com/c/1327809
Commit-Queue: Stephen Martinis <martiniss@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@chromium.org>
Cr-Commit-Position: refs/heads/master@{#609422}

docs/infra/cq.md

@@ -56,7 +56,9 @@ test suites which a given CL affects, and ensures that they all pass.
 
 ### What exactly does CQ run?
 
-CQ runs the jobs specified in [cq.cfg](../../infra/config/branch/cq.cfg).
+CQ runs the jobs specified in [cq.cfg](../../infra/config/branch/cq.cfg). See
+[`cq_builders.md`](cq_builders.md) for an auto generated file with links to
+information about the builders on the CQ.
 
 Some of these jobs are experimental. This means they are executed on a
 percentage of CQ builds, and the outcome of the build doesn't affect if the CL

infra/config/PRESUBMIT.py

@@ -3,9 +3,35 @@
 # found in the LICENSE file.
 
 
-def CheckChangeOnUpload(input_api, output_api):
-  return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api)
+def _CheckForTranslations(input_api, output_api):
+  del output_api
+
+  return []
+
+def _CommonChecks(input_api, output_api):
+  commands = []
+  # TODO(martiniss): Move this check to the root presubmit. It checks to ensure
+  # that path regexps in cq.cfg reference something locally.
+  for f in input_api.AffectedFiles():
+    local_path = f.LocalPath()
+    if local_path.endswith('cq.cfg'):
+      commands.append(
+        input_api.Command(
+          name='cq.cfg presubmit', cmd=[
+              input_api.python_executable, 'branch/cq_cfg_presubmit.py',
+              '--check'],
+          kwargs={}, message=output_api.PresubmitError),
+      )
 
+  results = []
+
+  results.extend(input_api.canned_checks.CheckChangedLUCIConfigs(
+      input_api, output_api))
+  results.extend(input_api.RunTests(commands))
+  return results
+
+def CheckChangeOnUpload(input_api, output_api):
+  return _CommonChecks(input_api, output_api)
 
 def CheckChangeOnCommit(input_api, output_api):
-  return input_api.canned_checks.CheckChangedLUCIConfigs(input_api, output_api)
+  return _CommonChecks(input_api, output_api)

infra/config/branch/cq.cfg

 # See http://luci-config.appspot.com/schemas/projects/refs:cq.cfg for the
 # documentation of this file format.
 
+# This file is also used to auto generate //docs/infra/cq_builders.md. If you
+# change this file, run //infra/config/branch/cq_cfg_presubmit.py, which will
+# generate that file. That script also requires that the builders in this file
+# remain sorted. The script is invoked via presubmit, and will complain if this
+# file is change but the documentation isn't.
+#
+# The auto generated file copies comments made to builders in this file. If you
+# comment on the line directly above a builder, that comment will get copied to
+# the documentation.
+#
+# The following comment will get copied.
+#
+# # This is a great builder!
+# builders { name: "chromium_presubmit" }
+#
+# The following comment will not get copied.
+#
+# # This is a ok builder!
+#
+# builders { name: "chromium_presubmit" }
+
 version: 1
 cq_status_url: "https://chromium-cq-status.appspot.com"
 git_repo_url: "https://chromium.googlesource.com/chromium/src"
@@ -30,13 +51,50 @@ verifiers {
   try_job {
     buckets {
       name: "luci.chromium.try"
-      builders { name: "android_arm64_dbg_recipe" }
+
+      #############################
+      # Always required builders. #
+      #############################
+
       builders { name: "android-binary-size" }
+      builders { name: "android-kitkat-arm-rel" }
+      builders { name: "android-marshmallow-arm64-rel" }
+      builders { name: "android_arm64_dbg_recipe" }
       builders { name: "android_clang_dbg_recipe" }
       builders { name: "android_compile_dbg" }
       builders { name: "android_cronet" }
-      builders { name: "android-kitkat-arm-rel" }
-      builders { name: "android-marshmallow-arm64-rel" }
+      builders { name: "cast_shell_android" }
+      builders { name: "cast_shell_linux" }
+      builders { name: "chromeos-amd64-generic-rel" }
+      builders { name: "chromeos-daisy-rel" }
+      builders {
+        name: "chromium_presubmit"
+        # Presubmit builder should be re-run every time CQ is triggered
+        # for last minute lint, OWNERS, etc checks.
+        disable_reuse: true
+      }
+      builders { name: "fuchsia_arm64" }
+      builders { name: "fuchsia_x64" }
+      builders { name: "ios-simulator" }
+      builders { name: "linux-chromeos-rel" }
+      builders { name: "linux-jumbo-rel" }
+      builders { name: "linux-libfuzzer-asan-rel" }
+      builders { name: "linux-ozone-rel" }
+      builders { name: "linux_chromium_asan_rel_ng" }
+      builders { name: "linux_chromium_compile_dbg_ng" }
+      builders { name: "linux_chromium_headless_rel" }
+      builders { name: "linux_chromium_rel_ng" }
+      builders { name: "linux_chromium_tsan_rel_ng" }
+      builders { name: "mac_chromium_compile_dbg_ng" }
+      builders { name: "mac_chromium_rel_ng" }
+      builders { name: "win10_chromium_x64_rel_ng" }
+      builders { name: "win7_chromium_rel_ng"}
+      builders { name: "win_chromium_compile_dbg_ng" }
+
+      ######################
+      # Optional builders. #
+      ######################
+
       builders {
         name: "android_optional_gpu_tests_rel"
         path_regexp: "cc/.+"
@@ -50,35 +108,12 @@ verifiers {
         path_regexp: "third_party/blink/renderer/modules/webgl/.+"
         path_regexp: "ui/gl/.+"
       }
-      builders { name: "cast_shell_android" }
-      builders { name: "cast_shell_linux" }
-      builders { name: "chromeos-amd64-generic-rel" }
-      builders { name: "chromeos-daisy-rel" }
-      builders {
-        name: "chromium_presubmit"
-        # Presubmit builder should be re-run every time CQ is triggered
-        # for last minute lint, OWNERS, etc checks.
-        disable_reuse: true
-      }
       builders {
         name: "closure_compilation"
         path_regexp: "components/offline_pages/resources/.+"
         path_regexp: "third_party/closure_compiler/.+"
         path_regexp: "third_party/polymer/.+"
       }
-      builders { name: "fuchsia_arm64" }
-      builders { name: "fuchsia_x64" }
-      # https://crbug.com/739556; make this non-experimental ASAP.
-      builders {
-        name: "ios-device"
-        experiment_percentage: 10
-      }
-      # https://crbug.com/739556
-      builders {
-        name: "ios-device-xcode-clang"
-        experiment_percentage: 10
-      }
-      builders { name: "ios-simulator" }
       builders {
         name: "ios-simulator-cronet"
         path_regexp: "components/cronet/.+"
@@ -90,41 +125,15 @@ verifiers {
         name: "ios-simulator-full-configs"
         path_regexp: "ios/.+"
       }
-      # https://crbug.com/739556
-      builders {
-        name: "ios-simulator-xcode-clang"
-        experiment_percentage: 10
-      }
       builders {
         name: "linux-blink-gen-property-trees"
         path_regexp: "third_party/WebKit/LayoutTests/FlagExpectations/enable-blink-gen-property-trees"
         path_regexp: "third_party/WebKit/LayoutTests/flag-specific/enable-blink-gen-property-trees/.+"
       }
-      builders { name: "linux-chromeos-rel" }
-      builders {
-        name: "linux-chromeos-compile-dbg"
-        experiment_percentage: 50
-      }
-      builders { name: "linux_chromium_asan_rel_ng" }
-      builders { name: "linux_chromium_compile_dbg_ng" }
       builders {
         name: "linux_chromium_dbg_ng"
         path_regexp: "build/.*check_gn_headers.*"
       }
-      builders { name: "linux_chromium_headless_rel" }
-      builders { name: "linux_chromium_rel_ng" }
-      builders { name: "linux_chromium_tsan_rel_ng" }
-      # https://crbug.com/833482
-      builders {
-        name: "linux-dcheck-off-rel"
-        experiment_percentage: 10
-      }
-      # https://crbug.com/855319
-      builders {
-        name: "linux-goma-rbe-staging-rel"
-        experiment_percentage: 20
-      }
-      builders { name: "linux-jumbo-rel" }
       builders {
         name: "linux_layout_tests_layout_ng"
         path_regexp: "third_party/Webkit/LayoutTests/FlagExpectations/enable-blink-features=LayoutNG"
@@ -141,14 +150,12 @@ verifiers {
         path_regexp: "third_party/blink/renderer/core/(svg|paint)/.+"
         path_regexp: "third_party/blink/renderer/platform/graphics/.+"
       }
-      builders { name: "linux-libfuzzer-asan-rel" }
       builders {
         name: "linux_mojo"
         path_regexp: "services/network/.+"
         path_regexp: "testing/buildbot/filters/mojo\\.fyi\\.network_.*"
         path_regexp: "third_party/WebKit/LayoutTests/FlagExpectations/enable-features=NetworkService"
       }
-      builders { name: "linux-ozone-rel" }
       builders {
         name: "linux_optional_gpu_tests_rel"
         path_regexp: "chrome/browser/vr/.+"
@@ -163,8 +170,6 @@ verifiers {
         name: "linux_vr"
         path_regexp: "chrome/browser/vr/.+"
       }
-      builders { name: "mac_chromium_compile_dbg_ng" }
-      builders { name: "mac_chromium_rel_ng" }
       builders {
         name: "mac_optional_gpu_tests_rel"
         path_regexp: "chrome/browser/vr/.+"
@@ -176,17 +181,6 @@ verifiers {
         path_regexp: "third_party/blink/renderer/modules/webgl/.+"
         path_regexp: "ui/gl/.+"
       }
-      builders {
-        name: "win-libfuzzer-asan-rel"
-        experiment_percentage: 100
-      }
-      builders { name: "win10_chromium_x64_rel_ng" }
-      builders {
-        name: "win7_chromium_rel_loc_exp"
-        experiment_percentage: 20
-      }
-      builders { name: "win7_chromium_rel_ng"}
-      builders { name: "win_chromium_compile_dbg_ng" }
       builders {
         name: "win_optional_gpu_tests_rel"
         path_regexp: "chrome/browser/vr/.+"
@@ -201,6 +195,48 @@ verifiers {
         path_regexp: "third_party/blink/renderer/platform/graphics/gpu/.+"
         path_regexp: "ui/gl/.+"
       }
+
+      ##########################
+      # Experimental builders. #
+      ##########################
+
+      # https://crbug.com/739556; make this non-experimental ASAP.
+      builders {
+        name: "ios-device"
+        experiment_percentage: 10
+      }
+      # https://crbug.com/739556
+      builders {
+        name: "ios-device-xcode-clang"
+        experiment_percentage: 10
+      }
+      # https://crbug.com/739556
+      builders {
+        name: "ios-simulator-xcode-clang"
+        experiment_percentage: 10
+      }
+      builders {
+        name: "linux-chromeos-compile-dbg"
+        experiment_percentage: 50
+      }
+      # https://crbug.com/833482
+      builders {
+        name: "linux-dcheck-off-rel"
+        experiment_percentage: 10
+      }
+      # https://crbug.com/855319
+      builders {
+        name: "linux-goma-rbe-staging-rel"
+        experiment_percentage: 20
+      }
+      builders {
+        name: "win-libfuzzer-asan-rel"
+        experiment_percentage: 100
+      }
+      builders {
+        name: "win7_chromium_rel_loc_exp"
+        experiment_percentage: 20
+      }
     }
 
     buckets {

infra/config/branch/cq_cfg_presubmit.py

+#!/usr/bin/env python
+# Copyright (c) 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import argparse
+import difflib
+import os
+import string
+import sys
+
+
+# Path to the root of the current chromium checkout.
+CHROMIUM_DIR = os.path.abspath(os.path.join(
+    os.path.dirname(__file__), '..', '..', '..'))
+
+
+MD_HEADER = """# List of CQ builders
+
+This page is auto generated using the script
+//infra/config/branch/cq_config_presubmit.py. Do not manually edit.
+
+[TOC]
+
+Each builder name links to that builder on Milo. The "Backing builders" links
+point to the file used to determine which configurations a builder should copy
+when running. These links might 404 or error; they are hard-coded right now,
+using common assumptions about how builders are configured.
+"""
+
+
+REQUIRED_HEADER = """
+These builders must pass before a CL may land."""
+
+
+OPTIONAL_HEADER = """These builders optionally run, depending on the files in a
+CL. For example, a CL which touches `//gpu/BUILD.gn` would trigger the builder
+`android_optional_gpu_tests_rel`, due to the `path_regexp` values for that
+builder."""
+
+
+EXPERIMENTAL_HEADER = """
+These builders are run on some percentage of builds. Their results are ignored
+by CQ. These are often used to test new configurations before they are added
+as required builders."""
+
+
+BUILDER_VIEW_URL = (
+    'https://ci.chromium.org/p/chromium/builders/luci.chromium.try/')
+
+
+CODE_SEARCH_BASE = 'https://cs.chromium.org/'
+
+
+TRYBOT_SOURCE_URL = CODE_SEARCH_BASE + 'search/?q=file:trybots.py+'
+
+
+CQ_CONFIG_LOCATION_URL = (
+    CODE_SEARCH_BASE + 'search/?q=package:%5Echromium$+file:cq.cfg+')
+
+
+REGEX_SEARCH_URL = CODE_SEARCH_BASE + 'search/?q=package:%5Echromium$+'
+
+
+def parse_text_proto_message(lines):
+  """Parses a text proto. LOW QUALITY, MAY EASILY BREAK.
+
+  If you really need to parse text protos, use the actual python library for
+  protobufs. This exists because the .proto file for cq.cfg lives in another
+  repository.
+  """
+  data = {}
+
+  linenum = 0
+  # Tracks the current comment. Gets cleared if there's a blank line. Is added
+  # to submessages, to allow for builders to contain comments.
+  current_comment = None
+  while linenum < len(lines):
+    line = lines[linenum].strip()
+    if not line:
+      current_comment = None
+      linenum += 1
+    elif line.startswith('#'):
+      if current_comment:
+        current_comment += '\n' + line[1:]
+      else:
+        current_comment = line[1:]
+      linenum += 1
+    elif '{' in line:
+      # Sub message. Put before the ':' clause so that it correctly handles one
+      # line messages.
+      end = linenum
+      count = 0
+      newlines = []
+      while end < len(lines):
+        inner_line = lines[end]
+        if '{' in inner_line:
+          count += 1
+        if '}' in inner_line:
+          count -= 1
+
+        if end == linenum:
+          newline = inner_line.split('{', 1)[1]
+          if count == 0:
+            newline = newline.split('}')[0]
+          newlines.append(newline)
+        elif count == 0:
+          newlines.append(inner_line.split('}')[0])
+        else:
+          newlines.append(inner_line)
+        end += 1
+        if count == 0:
+          break
+      name = line.split('{')[0].strip()
+      value = parse_text_proto_message(newlines)
+      if current_comment:
+        value['comment'] = current_comment
+        current_comment = None
+
+      if name in data:
+        data[name].append(value)
+      else:
+        data[name] = [value]
+      linenum = end
+    elif ':' in line:
+      # It's a field
+      name, value = line.split(':', 1)
+      value = value.strip()
+      if value.startswith('"'):
+        value = value.strip('"')
+
+      if name in data:
+        data[name].append(value)
+      else:
+        data[name] = [value]
+      linenum += 1
+    else:
+      raise ValueError('Invalid line (number %d):\n%s' % (linenum, line))
+
+  return data
+
+
+class BuilderList(object):
+  def __init__(self, builders):
+    self.builders = builders
+
+  def sort(self):
+    """Sorts the builder list.
+
+    Sorts the builders in place. Orders them into three groups: experimental,
+    required, and optional."""
+    self.builders.sort(key=lambda b: '%s|%s|%s' % (
+        'z' if b.get('experiment_percentage') else 'a',
+        'z' if b.get('path_regexp') else 'a',
+        b['name']))
+
+  def by_section(self):
+    required = []
+    experimental = []
+    optional = []
+    for b in self.builders:
+      # Don't handle if something is both optional and experimental
+      if b.get('path_regexp'):
+        optional.append(b)
+      elif b.get('experiment_percentage'):
+        experimental.append(b)
+      else:
+        required.append(b)
+
+    return required, optional, experimental
+
+
+class CQConfig(object):
+  def __init__(self, lines):
+    self._value = parse_text_proto_message(lines)
+
+  @staticmethod
+  def from_file(path):
+    with open(path) as f:
+      lines = f.readlines()
+
+    return CQConfig(lines)
+
+  @property
+  def version(self):
+    return int(self._value['version'][0])
+
+  def builder_list(self, pred=None):
+    """Returns a list of builders.
+
+    pred is a predicate used to decide if a builder should be returned. It takes
+    the bucket and builder as arguments."""
+    items = []
+    for bucket in (
+        self._value['verifiers'][0]['try_job'][0]['buckets']):
+      for b in bucket['builders']:
+        if pred and not pred(bucket, b):
+          continue
+        items.append(b)
+    return BuilderList(items)
+
+  def get_markdown_doc(self):
+    lines = []
+    for l in MD_HEADER.split('\n'):
+      lines.append(l)
+
+    bl = self.builder_list()
+    req, opt, exp = bl.by_section()
+    for title, header, builders in (
+        ('Required builders', REQUIRED_HEADER, req),
+        ('Optional builders', OPTIONAL_HEADER, opt),
+        ('Experimental builders', EXPERIMENTAL_HEADER, exp),
+    ):
+      lines.append('## %s' % title)
+      lines.append('')
+      for l in header.strip().split('\n'):
+        lines.append(l)
+      lines.append('')
+      for b in builders:
+        lines.append(
+            '* [%s](%s) ([`cq.cfg` entry](%s)) ([matching builders](%s))' % (
+            b['name'][0], BUILDER_VIEW_URL + b['name'][0],
+            CQ_CONFIG_LOCATION_URL + b['name'][0],
+            TRYBOT_SOURCE_URL + b['name'][0],))
+        lines.append('')
+        if 'comment' in b:
+          for l in b['comment'].split('\n'):
+            lines.append('  ' + l.strip())
+          lines.append('')
+        if 'path_regexp' in b:
+          lines.append('  Path regular expressions:')
+          for regex in b['path_regexp']:
+            regex_title = '//' + regex.lstrip('/')
+            url = None
+            if regex.endswith('.+'):
+              regex = regex[:-len('.+')]
+              if all(
+                  # Equals sign and dashes used by layout tests.
+                  c in string.ascii_letters + string.digits + '/-_='
+                  for c in regex):
+                # Assume the regex is targeting a single path, direct link to
+                # it. Check to make sure we don't have weird characters, like
+                # ()|, which could mean it's a regex.
+                url = CODE_SEARCH_BASE + 'chromium/src/' + regex
+            lines.append('    * [`%s`](%s)' % (
+                regex_title, url or REGEX_SEARCH_URL + 'file:' + regex))
+          lines.append('')
+        if 'experiment_percentage' in b:
+          lines.append('  * Experimental percentage: %s' % (
+              b['experiment_percentage'][0]))
+          lines.append('')
+      lines.append('')
+
+    return '\n'.join(lines)
+
+
+def main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+    '-c', '--check', action='store_true', help=
+    'Do consistency checks of cq.cfg and generated files. Used during'
+    'presubmit. Causes the tool to not generate any files.')
+  args = parser.parse_args()
+
+  exit_code = 0
+
+  cfg = CQConfig.from_file(os.path.join(
+      CHROMIUM_DIR, 'infra', 'config', 'branch', 'cq.cfg'))
+  if cfg.version != 1:
+    raise ValueError("Expected version 1, got %r" % cfg.version)
+
+  # Only force sorting on luci.chromium.try builders. Others should go away soon
+  # anyways...
+  bl = cfg.builder_list(lambda bucket, builder: bucket == 'luci.chromium.try')
+  names = [b['name'][0] for b in bl.builders]
+  bl.sort() # Changes the bl, so the next line is sorted.
+  sorted_names = [b['name'][0] for b in bl.builders]
+  if sorted_names != names:
+    print 'ERROR: cq.cfg is unsorted.',
+    if args.check:
+      print
+    else:
+      print ' Please sort as follows:'
+      for line in difflib.unified_diff(
+          names,
+          sorted_names, fromfile='current', tofile='sorted'):
+        print line
+    exit_code = 1
+
+  if args.check:
+    # TODO(martiniss): Add a check for path_regexp, to make sure they're valid
+    # paths.
+    with open(os.path.join(
+        CHROMIUM_DIR, 'docs', 'infra', 'cq_builders.md')) as f:
+      if cfg.get_markdown_doc() != f.read():
+        print (
+            'Markdown file is out of date. Please run '
+            '`//infra/config/branch/cq_cfg_presubmit.py to regenerate the '
+            'docs.')
+        exit_code = 1
+  else:
+    with open(os.path.join(
+        CHROMIUM_DIR, 'docs', 'infra', 'cq_builders.md'), 'w') as f:
+      f.write(cfg.get_markdown_doc())
+
+  return exit_code
+
+
+if __name__ == '__main__':
+  sys.exit(main())