Add a library for altering starlark execution based on the branch.

Separating the branch configuration by file has been rather unwieldy and
would only get worse when it is necessary to maintain a 3rd subset of
builders for LTS. This puts the information about what branches the LUCI
resources are applicable to with the resource.

Bug: 1119446
Change-Id: I1c8622b7084fd332ae16fcdc9afe5d6e30dc1811
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2365915
Commit-Queue: Garrett Beaty <gbeaty@chromium.org>
Reviewed-by: Erik Staab <estaab@chromium.org>
Cr-Commit-Position: refs/heads/master@{#800627}

infra/config/lib/branches.star

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Library containing utilities for providing branch-specific definitions.
+
+The module provide the `branches` struct which provides access to versions of
+a subset of luci functions with an additional `branch_selector` keyword argument
+that controls what branches the definition is actually executed for. If
+`branch_selector` doesn't match the current branch as determined by values on
+the `settings` struct in '//project.star', then the resource is not defined. The
+`branch_selector argument` can be one of the following constants:
+* MAIN_ONLY - The resource is defined only for main/master/trunk
+    [`settings.is_master`]
+* STANDARD_RELEASES - The resource is defined for main/master/trunk and beta and
+    stable branches
+    [`settings.is_master and not settings.is_lts_branch`]
+* ALL_RELEASES - The resource is defined for main/master/trunk, beta and stable
+    branches and LTC/LTS branches
+    [`True`]
+The constants are also accessible via the `branches` struct.
+
+For other uses cases where execution needs to vary by branch, the following are
+also accessible via the `branches` struct:
+* matches - Allows library code to be written that takes branch-specific
+    behavior.
+* value - Allows for providing different values between main/master/trunk and
+    branches.
+* exec - Allows for conditionally executing starlark modules.
+"""
+
+load("//project.star", "settings")
+
+def _branch_selector(tag):
+    return struct(__branch_selector__ = tag)
+
+MAIN_ONLY = _branch_selector("MAIN_ONLY")
+STANDARD_RELEASES = _branch_selector("STANDARD_RELEASES")
+ALL_RELEASES = _branch_selector("ALL_RELEASES")
+
+_BRANCH_SELECTORS = (MAIN_ONLY, STANDARD_RELEASES, ALL_RELEASES)
+
+def _matches(branch_selector):
+    """Returns whether `branch_selector` matches the project settings."""
+    if branch_selector == MAIN_ONLY:
+        return settings.is_master
+    if branch_selector == STANDARD_RELEASES:
+        return settings.is_master or not settings.is_lts_branch
+    if branch_selector == ALL_RELEASES:
+        return True
+    fail("branch_selector must be one of {}, got {!r}".format(_BRANCH_SELECTORS, branch_selector))
+
+def _value(*, for_main = None, for_branches = None):
+    """Provide a value that varies between main/master/trunk and branches.
+
+    If the current project settings indicate that this is main/master/trunk,
+    then `for_main` will be returned. Otherwise, `for_branches` will be
+    returned.
+    """
+    return for_main if settings.is_master else for_branches
+
+def _exec(module, *, branch_selector = MAIN_ONLY):
+    """Execute `module` if `branch_selector` matches the project settings."""
+    if not _matches(branch_selector):
+        return
+    exec(module)
+
+def _make_branch_conditional(fn):
+    def conditional_fn(*args, branch_selector = MAIN_ONLY, **kwargs):
+        if not _matches(branch_selector):
+            return
+        fn(*args, **kwargs)
+
+    return conditional_fn
+
+branches = struct(
+    MAIN_ONLY = MAIN_ONLY,
+    STANDARD_RELEASES = STANDARD_RELEASES,
+    ALL_RELEASES = ALL_RELEASES,
+    matches = _matches,
+    exec = _exec,
+    value = _value,
+
+    # Make conditional versions of luci functions that define resources
+    # This does not include any of the service configurations
+    # This also does not include any functions such as recipe that don't
+    # generate config unless they're referred to; it doesn't cause a problem
+    # if they're not referred to
+    **{a: _make_branch_conditional(getattr(luci, a)) for a in (
+        "realm",
+        "binding",
+        "bucket",
+        "builder",
+        "gitiles_poller",
+        "list_view",
+        "list_view_entry",
+        "console_view",
+        "console_view_entry",
+        "external_console_view",
+        "cq_group",
+        "cq_tryjob_verifier",
+    )}
+)

infra/config/lib/builders.star

@@ -27,6 +27,7 @@ through `builders.cpu`, `builders.os` and `builders.goma` respectively.
 
 load("//project.star", "settings")
 load("./args.star", "args")
+load("./branches.star", "branches")
 
 ################################################################################
 # Constants for use with the builder function                                  #
@@ -273,6 +274,7 @@ defaults = args.defaults(
 def builder(
         *,
         name,
+        branch_selector = branches.MAIN_ONLY,
         bucket = args.DEFAULT,
         executable = args.DEFAULT,
         triggered_by = args.DEFAULT,
@@ -317,6 +319,8 @@ def builder(
 
     Arguments:
       * name - name of the builder, will show up in UIs and logs. Required.
+      * branch_selector - A branch selector value controlling whether the
+        builder definition is executed. See branches.star for more information.
       * bucket - a bucket the build is in, see luci.bucket(...) rule. Required
         (may be specified by module-level default).
       * executable - an executable to run, e.g. a luci.recipe(...). Required (may
@@ -527,8 +531,9 @@ def builder(
     if triggered_by != args.COMPUTE:
         kwargs["triggered_by"] = triggered_by
 
-    return luci.builder(
+    return branches.builder(
         name = name,
+        branch_selector = branch_selector,
         dimensions = dimensions,
         properties = properties,
         resultdb_settings = resultdb.settings(

infra/config/lib/ci.star

@@ -16,8 +16,9 @@ to set the default value. Can also be accessed through `ci.defaults`.
 load("@stdlib//internal/graph.star", "graph")
 load("@stdlib//internal/luci/common.star", "keys")
 load("//project.star", "settings")
-load("./builders.star", "builders")
 load("./args.star", "args")
+load("./branches.star", "branches")
+load("./builders.star", "builders")
 
 defaults = args.defaults(
     extends = builders.defaults,
@@ -29,7 +30,10 @@ defaults = args.defaults(
     repo = None,
 )
 
-def declare_bucket(milestone_vars):
+def declare_bucket(milestone_vars, *, branch_selector = branches.MAIN_ONLY):
+    if not branches.matches(branch_selector):
+        return
+
     luci.bucket(
         name = milestone_vars.ci_bucket,
         acls = [
@@ -63,6 +67,7 @@ def declare_bucket(milestone_vars):
     ):
         ci.overview_console_view(
             name = name,
+            branch_selector = branch_selector,
             header = "//chromium-header.textpb",
             repo = "https://chromium.googlesource.com/chromium/src",
             refs = [milestone_vars.ref],
@@ -310,7 +315,7 @@ def ordering(*, short_names = None, categories = None):
         categories = categories or [],
     )
 
-def console_view(*, name, ordering = None, **kwargs):
+def console_view(*, name, branch_selector = branches.MAIN_ONLY, ordering = None, **kwargs):
     """Create a console view, optionally providing an entry ordering.
 
     Args:
@@ -341,6 +346,9 @@ def console_view(*, name, ordering = None, **kwargs):
         `luci.console_view`. The header and repo arguments support
          module-level defaults.
     """
+    if not branches.matches(branch_selector):
+        return
+
     kwargs["header"] = defaults.get_value_from_kwargs("header", kwargs)
     kwargs["repo"] = defaults.get_value_from_kwargs("repo", kwargs)
     luci.console_view(
@@ -353,7 +361,7 @@ def console_view(*, name, ordering = None, **kwargs):
         ordering = ordering or {},
     )
 
-def overview_console_view(*, name, top_level_ordering, **kwargs):
+def overview_console_view(*, name, top_level_ordering, branch_selector = branches.MAIN_ONLY, **kwargs):
     """Create an overview console view.
 
     An overview console view is a console view that contains a subset of
@@ -406,6 +414,7 @@ def console_view_entry(*, category = None, short_name = None):
 def ci_builder(
         *,
         name,
+        branch_selector = branches.MAIN_ONLY,
         add_to_console_view = args.DEFAULT,
         console_view = args.DEFAULT,
         main_console_view = args.DEFAULT,
@@ -418,6 +427,9 @@ def ci_builder(
 
     Arguments:
       name - name of the builder, will show up in UIs and logs. Required.
+      branch_selector - A branch selector value controlling whether the
+        builder definition is executed. See branches.star for more
+        information.
       add_to_console_view - A bool indicating whether an entry should be
         created for the builder in the console identified by
         `console_view`. Supports a module-level default that defaults to
@@ -444,6 +456,8 @@ def ci_builder(
         'chromium-tree-closer' config in notifiers.star for the full criteria.
       notifies - Any extra notifiers to attach to this builder.
     """
+    if not branches.matches(branch_selector):
+        return
 
     # Branch builders should never close the tree, only builders from the main
     # "ci" bucket.
@@ -453,8 +467,9 @@ def ci_builder(
 
     # Define the builder first so that any validation of luci.builder arguments
     # (e.g. bucket) occurs before we try to use it
-    ret = builders.builder(
+    builders.builder(
         name = name,
+        branch_selector = branch_selector,
         resultdb_bigquery_exports = [resultdb.export_test_results(
             bq_table = "luci-resultdb.chromium.ci_test_results",
         )],
@@ -487,12 +502,15 @@ def ci_builder(
                     short_name = console_view_entry.short_name,
                 )
 
+            overview_console_category = console_view
+            if console_view_entry.category:
+                overview_console_category = "|".join([console_view, console_view_entry.category])
             main_console_view = defaults.get_value("main_console_view", main_console_view)
             if main_console_view:
                 luci.console_view_entry(
                     builder = builder,
                     console_view = main_console_view,
-                    category = "|".join([console_view, console_view_entry.category]),
+                    category = overview_console_category,
                     short_name = console_view_entry.short_name,
                 )
 
@@ -504,12 +522,10 @@ def ci_builder(
                 luci.console_view_entry(
                     builder = builder,
                     console_view = cq_mirrors_console_view,
-                    category = "|".join([console_view, console_view_entry.category]),
+                    category = overview_console_category,
                     short_name = console_view_entry.short_name,
                 )
 
-    return ret
-
 def android_builder(
         *,
         name,

infra/config/lib/try.star

@@ -19,8 +19,9 @@ to set the default value. Can also be accessed through `try_.defaults`.
 
 load("@stdlib//internal/graph.star", "graph")
 load("@stdlib//internal/luci/common.star", "keys")
-load("./builders.star", "builders")
 load("./args.star", "args")
+load("./branches.star", "branches")
+load("./builders.star", "builders")
 
 DEFAULT_EXCLUDE_REGEXPS = [
     # Contains documentation that doesn't affect the outputs
@@ -37,7 +38,10 @@ defaults = args.defaults(
     main_list_view = None,
 )
 
-def declare_bucket(milestone_vars):
+def declare_bucket(milestone_vars, *, branch_selector = branches.MAIN_ONLY):
+    if not branches.matches(branch_selector):
+        return
+
     luci.bucket(
         name = milestone_vars.try_bucket,
         acls = [
@@ -88,6 +92,7 @@ def declare_bucket(milestone_vars):
 
     try_.list_view(
         name = milestone_vars.main_list_view_name,
+        branch_selector = branch_selector,
         title = milestone_vars.main_list_view_title,
     )
 
@@ -146,8 +151,11 @@ def _sort_console_entries(ctx):
 
 lucicfg.generator(_sort_console_entries)
 
-def list_view(*, name, **kwargs):
-    ret = luci.list_view(
+def list_view(*, name, branch_selector = branches.MAIN_ONLY, **kwargs):
+    if not branches.matches(branch_selector):
+        return
+
+    luci.list_view(
         name = name,
         **kwargs
     )
@@ -156,8 +164,6 @@ def list_view(*, name, **kwargs):
         console_name = name,
     )
 
-    return ret
-
 def tryjob(
         *,
         disable_reuse = None,
@@ -194,6 +200,7 @@ def tryjob(
 def try_builder(
         *,
         name,
+        branch_selector = branches.MAIN_ONLY,
         add_to_list_view = args.DEFAULT,
         cq_group = args.DEFAULT,
         list_view = args.DEFAULT,
@@ -204,6 +211,9 @@ def try_builder(
 
     Arguments:
       name - name of the builder, will show up in UIs and logs. Required.
+      branch_selector - A branch selector value controlling whether the
+        builder definition is executed. See branches.star for more
+        information.
       add_to_list_view - A bool indicating whether an entry should be
         created for the builder in the console identified by
         `list_view`. Supports a module-level default that defaults to
@@ -221,11 +231,14 @@ def try_builder(
       tryjob - A struct containing the details of the tryjob verifier for the
         builder, obtained by calling the `tryjob` function.
     """
+    if not branches.matches(branch_selector):
+        return
 
     # Define the builder first so that any validation of luci.builder arguments
     # (e.g. bucket) occurs before we try to use it
-    ret = builders.builder(
+    builders.builder(
         name = name,
+        branch_selector = branch_selector,
         resultdb_bigquery_exports = [resultdb.export_test_results(
             bq_table = "luci-resultdb.chromium.try_test_results",
         )],
@@ -281,8 +294,6 @@ def try_builder(
             list_view = main_list_view,
         )
 
-    return ret
-
 def blink_builder(*, name, goma_backend = None, **kwargs):
     return try_builder(
         name = name,

infra/config/notifiers.star

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+load("//lib/branches.star", "branches")
+
 luci.notifier(
     name = "chromesec-lkgr-failures",
     on_status_change = True,
@@ -48,23 +50,48 @@ TREE_CLOSING_STEPS = [
     "update",
 ]
 
-luci.tree_closer(
+# This results in a notifier with no recipients, so nothing will actually be
+# notified. This still creates a "notifiable" that can be passed to the notifies
+# argument of a builder, so conditional logic doesn't need to be used when
+# setting the argument and erroneous tree closure notifications won't be sent
+# for failures on branches.
+def _empty_notifier(*, name):
+    luci.notifier(
+        name = name,
+        on_new_status = ["INFRA_FAILURE"],
+    )
+
+def tree_closer(*, name, tree_status_host, **kwargs):
+    if branches.matches(branches.MAIN_ONLY):
+        luci.tree_closer(
+            name = name,
+            tree_status_host = tree_status_host,
+            **kwargs
+        )
+    else:
+        _empty_notifier(name = name)
+
+tree_closer(
     name = "chromium-tree-closer",
     tree_status_host = "chromium-status.appspot.com",
     failed_step_regexp = TREE_CLOSING_STEPS,
 )
 
-luci.tree_closer(
+tree_closer(
     name = "close-on-any-step-failure",
     tree_status_host = "chromium-status.appspot.com",
 )
 
-def tree_closure_notifier(**kwargs):
-    return luci.notifier(
-        on_occurrence = ["FAILURE"],
-        failed_step_regexp = TREE_CLOSING_STEPS,
-        **kwargs
-    )
+def tree_closure_notifier(*, name, **kwargs):
+    if branches.matches(branches.MAIN_ONLY):
+        luci.notifier(
+            name = name,
+            on_occurrence = ["FAILURE"],
+            failed_step_regexp = TREE_CLOSING_STEPS,
+            **kwargs
+        )
+    else:
+        _empty_notifier(name = name)
 
 tree_closure_notifier(
     name = "chromium-tree-closer-email",