Make sure jobs for CI builders work with or without bucket prefix.

When the number of builders with the same name goes from 1 to >1 or
vice-versa, transient infra failures occur because the scheduler
configuration will be updated while there are already builds running
that will attempt to trigger the old job name. This change removes that
as a problem.

Change-Id: I1547659f61912ef758edb53f86f2092e723a7c54
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2107805
Commit-Queue: Garrett Beaty <gbeaty@chromium.org>
Auto-Submit: Garrett Beaty <gbeaty@chromium.org>
Reviewed-by: Aaron Gable <agable@chromium.org>
Cr-Commit-Position: refs/heads/master@{#751409}

infra/config/generators/scheduler-bucketed-jobs.star

+def _ensure_ci_jobs_can_be_bucketed_or_not(ctx):
+  cfg = ctx.output['luci-scheduler.cfg']
+
+  ci_jobs = {}
+  for j in cfg.job:
+    bucket = j.buildbucket.bucket
+    if bucket != 'luci.chromium.ci':
+      continue
+
+    builder = j.buildbucket.builder
+    if builder in ci_jobs:
+      fail('Multiple jobs defined for CI builder ci/{}'.format(builder))
+
+    ci_jobs[builder] = j
+
+  for builder, job in ci_jobs.items():
+    new_job = proto.from_wirepb(proto.message_type(job), proto.to_wirepb(job))
+    new_job.id = builder if job.id != builder else 'ci-{}'.format(builder)
+    cfg.job.append(new_job)
+
+  jobs = sorted(cfg.job, key=lambda j: j.id)
+  cfg.job.clear()
+  cfg.job.extend(jobs)
+
+lucicfg.generator(_ensure_ci_jobs_can_be_bucketed_or_not)

infra/config/generators/scheduler-noop-jobs.star

@@ -58,11 +58,4 @@ def _add_noop_jobs(ctx):
   for j in _GPU_NOOP_JOBS + _ANDROID_TEST_NOOP_JOBS:
     cfg.job.append(j)
 
-def _munge_trunk_only_jobs(ctx):
-  cfg = ctx.output['luci-scheduler.cfg']
-  for j in cfg.job:
-    if j.id in _ANDROID_NON_BRANCHED_TESTERS:
-      j.id = 'ci-' + j.id
-
 lucicfg.generator(_add_noop_jobs)
-lucicfg.generator(_munge_trunk_only_jobs)

infra/config/main.star

@@ -145,6 +145,14 @@ exec('//consoles/tryserver.chromium.win.star')
 exec('//notifiers.star')
 
 exec('//generators/cq-builders-md.star')
+# This should be exec'ed before exec'ing scheduler-noop-jobs.star because
+# attempting to read the buildbucket field that is not set for the noop jobs
+# actually causes an empty buildbucket message to be set
+# TODO(https://crbug.com/1062385) The automatic generation of job IDs causes
+# problems when the number of builders with the same name goes from 1 to >1 or
+# vice-versa. This generator makes sure both the bucketed and non-bucketed IDs
+# work so that there aren't transient failures when the configuration changes
+exec('//generators/scheduler-bucketed-jobs.star')
 # TODO(https://crbug.com/966115) Run the generator to set the fallback field for
 # the empty CQ group until it's exposed in lucicfg or there is a better way to
 # create a CQ group for all of the canary branches