Migrate commit-queue.cfg to starlark.

Bug: 1011908
Change-Id: I95190f8f0ecbad7a5ae556145e92e9b5094041f9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1864419Reviewed-by: Stephen Martinis <martiniss@chromium.org>
Commit-Queue: Garrett Beaty <gbeaty@chromium.org>
Cr-Commit-Position: refs/heads/master@{#706669}

docs/infra/cq.md

@@ -63,8 +63,9 @@ The Chromium CQ supports a variety of options that can change what it checks.
 ### What exactly does the CQ run?
 
 CQ runs the jobs specified in [commit-queue.cfg][2]. See
-[`cq_builders.md`](cq_builders.md) for an auto generated file with links to
-information about the builders on the CQ.
+[`cq-builders.md`](https://chromium.googlesource.com/chromium/src/+/master/src/infra/config/generated/cq-builders.md)
+for an auto generated file with links to information about the builders on the
+CQ.
 
 Some of these jobs are experimental. This means they are executed on a
 percentage of CQ builds, and the outcome of the build doesn't affect if the CL

infra/config/PRESUBMIT.py

@@ -31,26 +31,11 @@ def _CommonChecks(input_api, output_api):
             '--check'],
         kwargs={}, message=output_api.PresubmitError))
 
-  if 'infra/config/commit-queue.cfg' in input_api.LocalPaths():
-    commands.append(
-      input_api.Command(
-        name='commit-queue.cfg presubmit', cmd=[
-            input_api.python_executable, input_api.os_path.join(
-                'cq_cfg_presubmit.py'),
-            '--check'],
-        kwargs={}, message=output_api.PresubmitError),
-    )
-
   commands.extend(input_api.canned_checks.CheckLucicfgGenOutput(
       input_api, output_api, 'main.star'))
   commands.extend(input_api.canned_checks.CheckLucicfgGenOutput(
       input_api, output_api, 'dev.star'))
 
-  commands.extend(input_api.canned_checks.GetUnitTestsRecursively(
-      input_api, output_api,
-      input_api.os_path.join(input_api.PresubmitLocalPath()),
-      whitelist=[r'.+_unittest\.py$'], blacklist=[]))
-
   results = []
 
   results.extend(input_api.RunTests(commands))

infra/config/buckets/try.star

@@ -24,6 +24,8 @@ luci.bucket(
     ],
 )
 
+exec('./try/cq.star')
+
 luci.recipe.defaults.cipd_package.set('infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build')
 
 defaults.bucket.set('try')

infra/config/cq-builders-md.star

+_MD_HEADER = """\
+<!-- Auto-generated by lucicfg (via cq-builders-md.star). -->
+<!-- Do not modify manually. -->
+
+# List of CQ builders
+
+[TOC]
+
+Each builder name links to that builder on Milo. The "matching builders" links
+point to the file used to determine which configurations a builder should copy
+when running. These links might 404 or error; they are hard-coded right now,
+using common assumptions about how builders are configured.
+"""
+
+_REQUIRED_HEADER = """\
+These builders must pass before a CL may land.
+"""
+
+_OPTIONAL_HEADER = """\
+These builders optionally run, depending on the files in a CL. For example, a CL
+which touches `//gpu/BUILD.gn` would trigger the builder
+`android_optional_gpu_tests_rel`, due to the `location_regexp` values for that
+builder.
+"""
+
+_EXPERIMENTAL_HEADER = """\
+These builders are run on some percentage of builds. Their results are ignored
+by CQ. These are often used to test new configurations before they are added
+as required builders.
+"""
+
+_TRY_BUILDER_VIEW_URL = 'https://ci.chromium.org/p/chromium/builders/try'
+
+_REGEX_PREFIX = '.+/[+]/'
+
+
+def _get_main_config_group_builders(ctx):
+  cq_cfg = ctx.output['commit-queue.cfg']
+
+  for c in cq_cfg.config_groups:
+    if len(c.gerrit) != 1:
+      continue
+
+    gerrit = c.gerrit[0]
+    if len(gerrit.projects) != 1:
+      continue
+
+    project = gerrit.projects[0]
+    if len(project.ref_regexp) != 1:
+      continue
+
+    if (project.name == 'chromium/src'
+        # Repeated proto fields have an internal type that won't compare equal
+        # to a list, so convert it
+        and list(project.ref_regexp) == ['refs/heads/.+']):
+      return c.verifiers.tryjob.builders
+
+  fail('Could not find the main CQ group')
+
+
+def _group_builders_by_section(builders):
+  required = []
+  experimental = []
+  optional = []
+
+  for builder in builders:
+    if builder.experiment_percentage:
+      experimental.append(builder)
+    elif builder.location_regexp or builder.location_regexp_exclude:
+      optional.append(builder)
+    else:
+      required.append(builder)
+
+  return struct(
+      required = required,
+      experimental = experimental,
+      optional = optional,
+  )
+
+
+def _codesearch_query(*atoms, package=None):
+  query = ['https://cs.chromium.org/search?q=']
+  if package != None:
+    query.append('package:%5E')  # %5E -> encoded ^
+    query.append(package)
+    query.append('$')
+  for atom in atoms:
+    query.append('+')
+    query.append(atom)
+  return ''.join(query)
+
+
+def _get_regex_line_details(regex):
+  if regex.startswith(_REGEX_PREFIX):
+    regex = regex[len(_REGEX_PREFIX):]
+  title = '//' + regex.lstrip('/')
+  if regex.endswith('.+'):
+    regex = regex[:-len('.+')]
+
+  url = _codesearch_query('file:' + regex, package='chromium')
+  # If the regex doesn't have any interesting characters that might be part of a
+  # regex, assume the regex is targeting a single path and direct link to it
+  # Equals sign and dashes used by layout tests
+  if all([c.isalnum() or c in '/-_=' for c in regex.codepoints()]):
+    url = 'https://cs.chromium.org/chromium/src/' + regex
+
+  return struct(
+      title=title,
+      url=url,
+  )
+
+
+def _generate_cq_builders_md(ctx):
+  builders = _get_main_config_group_builders(ctx)
+
+  builders_by_section = _group_builders_by_section(builders)
+
+  lines = [_MD_HEADER]
+
+  for title, header, section in (
+      ('Required builders', _REQUIRED_HEADER, 'required'),
+      ('Optional builders', _OPTIONAL_HEADER, 'optional'),
+      ('Experimental builders', _EXPERIMENTAL_HEADER, 'experimental'),
+  ):
+    builders = getattr(builders_by_section, section)
+    if not builders:
+      continue
+
+    lines.append('## %s' % title)
+    lines.append(header)
+
+    for b in builders:
+      name = b.name.rsplit('/', 1)[-1]
+      lines.append((
+          '* [{name}]({try_builder_view}/{name}) '
+          + '([definition]({definition_query}+{name})) '
+          + '([matching builders]({trybot_query}+{name}))'
+      ).format(
+          name=name,
+          try_builder_view=_TRY_BUILDER_VIEW_URL,
+          definition_query=_codesearch_query(
+              'file:/cq.star$', '-file:/beta/', '-file:/stable/',
+              package='chromium'),
+          trybot_query=_codesearch_query('file:trybots.py'),
+      ))
+
+      if b.experiment_percentage:
+        lines.append('  * Experiment percentage: {percentage}'.format(
+            percentage=b.experiment_percentage,
+        ))
+
+      for attr, regexp_header in (
+          ('location_regexp', 'Path regular expressions:'),
+          ('location_regexp_exclude', 'Path exclude regular expressions:'),
+      ):
+        regexes = getattr(b, attr)
+        if not regexes:
+          continue
+        lines.append('')
+        lines.append('  ' + regexp_header)
+        for regex in regexes:
+          regex_line_details = _get_regex_line_details(regex)
+          lines.append('  * [`{title}`]({url})'.format(
+              title=regex_line_details.title,
+              url=regex_line_details.url,
+          ))
+
+      lines.append('')
+
+    lines.append('')
+
+  ctx.output['cq-builders.md'] = '\n'.join(lines)
+
+lucicfg.generator(_generate_cq_builders_md)

infra/config/cq_cfg_presubmit_unittest.py

-#!/usr/bin/env vpython
-# Copyright (c) 2018 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-"""Tests for cq_cfg_presubmit."""
-
-import mock
-import os
-import unittest
-
-import cq_cfg_presubmit
-
-
-class CqCfgPresubmitTest(unittest.TestCase):
-  def test_verify_location_regexp_exists(self):
-    with mock.patch('cq_cfg_presubmit.os.path.exists') as exists:
-      exists.side_effect = [True]
-      self.assertTrue(cq_cfg_presubmit.verify_location_regexps([
-          cq_cfg_presubmit.REGEX_PREFIX + 'simple/file',
-      ]))
-
-  def test_verify_location_regexp_os_walk_found(self):
-    with mock.patch('cq_cfg_presubmit.os.walk') as walk:
-      walk.side_effect = [(
-          (os.path.join(cq_cfg_presubmit.CHROMIUM_DIR, 'random'),
-            None, ['test.txt'],),
-          (os.path.join(cq_cfg_presubmit.CHROMIUM_DIR, 'simple', 'file'),
-           None, ['test.txt'],),
-      )]
-      with mock.patch('cq_cfg_presubmit.os.path.exists') as exists:
-        exists.side_effect = [False]
-        self.assertTrue(cq_cfg_presubmit.verify_location_regexps([
-            cq_cfg_presubmit.REGEX_PREFIX + 'simple/file/.+',
-        ], False))
-
-  def test_verify_location_regexp_os_walk_not_found(self):
-    with mock.patch('cq_cfg_presubmit.os.walk') as walk:
-      walk.side_effect = [(
-          (os.path.join(cq_cfg_presubmit.CHROMIUM_DIR, 'random'),
-            None, ['test.txt'],),
-      )]
-      with mock.patch('cq_cfg_presubmit.os.path.exists') as exists:
-        exists.side_effect = [False]
-        self.assertFalse(cq_cfg_presubmit.verify_location_regexps([
-            cq_cfg_presubmit.REGEX_PREFIX + 'simple/file/.+',
-        ], False))
-
-
-if __name__ == '__main__':
-  unittest.main()

infra/config/main.star

@@ -7,6 +7,7 @@ lucicfg.config(
     config_dir = 'generated',
     tracked_files = [
         'commit-queue.cfg',
+        'cq-builders.md',
         'cr-buildbucket.cfg',
         'luci-logdog.cfg',
         'luci-milo.cfg',
@@ -21,7 +22,6 @@ lucicfg.config(
 # Copy the not-yet migrated files to the generated outputs
 # TODO(https://crbug.com/1011908) Migrate the configuration in these files to starlark
 [lucicfg.emit(dest = f, data = io.read_file(f)) for f in (
-    'commit-queue.cfg',
     'luci-milo.cfg',
     # TODO(https://crbug.com/1015148) lucicfg generates luci-notify.cfg very
     # differently from our hand-written file and doesn't do any normalization
@@ -61,6 +61,12 @@ luci.project(
     ],
 )
 
+luci.cq(
+    submit_max_burst = 2,
+    submit_burst_delay = time.minute,
+    status_host = 'chromium-cq-status.appspot.com',
+)
+
 luci.logdog(
     gs_bucket = 'chromium-luci-logdog',
 )
@@ -70,3 +76,5 @@ exec('//buckets/findit.star')
 exec('//buckets/try.star')
 exec('//buckets/webrtc.star')
 exec('//buckets/webrtc.fyi.star')
+
+exec('//cq-builders-md.star')