Add Swarming realms configuration.

Based on pools.cfg in infradata/config. Not really used by anything
yet.

One complication is per-milestone projects: we "fork" builders, but
not Swarming pools. So some chunks of configuration should be defined
only on the main branch.

R=gbeaty@chromium.org, estaab@chromium.org, iannucci@chromium.org
BUG=1123142

Change-Id: I4850e1be0efc6215a0bc58f3ad707dbdc3a1e473
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2401814Reviewed-by: Erik Staab <estaab@chromium.org>
Reviewed-by: Garrett Beaty <gbeaty@chromium.org>
Reviewed-by: Robbie Iannucci <iannucci@chromium.org>
Commit-Queue: Vadim Shtayura <vadimsh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#806273}

infra/config/generated/realms.cfg

@@ -26,6 +26,29 @@ realms {
     role: "role/scheduler.reader"
     principals: "group:all"
   }
+  bindings {
+    role: "role/swarming.poolOwner"
+    principals: "group:project-chromium-admins"
+  }
+  bindings {
+    role: "role/swarming.poolUser"
+    principals: "group:google/luci-task-force@google.com"
+    principals: "group:mdb/chrome-troopers"
+  }
+  bindings {
+    role: "role/swarming.poolViewer"
+    principals: "group:all"
+  }
+  bindings {
+    role: "role/swarming.taskServiceAccount"
+    principals: "group:project-chromium-test-task-accounts"
+    principals: "user:ios-isolated-tester@chops-service-accounts.iam.gserviceaccount.com"
+  }
+  bindings {
+    role: "role/swarming.taskTriggerer"
+    principals: "group:google/luci-task-force@google.com"
+    principals: "group:mdb/chrome-troopers"
+  }
 }
 realms {
   name: "ci"
@@ -104,6 +127,10 @@ realms {
     principals: "group:findit-tryjob-access"
     principals: "user:luci-scheduler@appspot.gserviceaccount.com"
   }
+  bindings {
+    role: "role/swarming.taskTriggerer"
+    principals: "group:project-findit-owners"
+  }
 }
 realms {
   name: "goma"
@@ -124,6 +151,58 @@ realms {
     principals: "group:project-chromium-ci-schedulers"
   }
 }
+realms {
+  name: "pools/ci"
+  bindings {
+    role: "role/swarming.poolUser"
+    principals: "project:chromium-m85"
+    principals: "project:chromium-m86"
+  }
+}
+realms {
+  name: "pools/findit"
+  bindings {
+    role: "role/swarming.poolUser"
+    principals: "group:project-findit-owners"
+  }
+}
+realms {
+  name: "pools/tests"
+  bindings {
+    role: "role/swarming.poolUser"
+    principals: "group:project-chromium-ci-task-accounts"
+    principals: "group:project-chromium-findit-task-accounts"
+    principals: "group:project-chromium-try-task-accounts"
+    principals: "group:project-chromium-tryjob-access"
+    principals: "group:project-devtools-frontend-ci-task-accounts"
+    principals: "group:project-devtools-frontend-try-task-accounts"
+    principals: "group:project-v8-ci-task-accounts"
+    principals: "group:project-v8-try-task-accounts"
+    principals: "group:project-webrtc-ci-task-accounts"
+    principals: "group:project-webrtc-try-task-accounts"
+    principals: "group:service-account-chromeperf"
+    principals: "user:chromium-swarm-bots@skia-swarming-bots.iam.gserviceaccount.com"
+    principals: "user:skia-external-ct-skps@skia-swarming-bots.iam.gserviceaccount.com"
+  }
+}
+realms {
+  name: "pools/tests-mac-arm64"
+  bindings {
+    role: "role/swarming.poolUser"
+    principals: "group:project-chromium-ci-task-accounts"
+    principals: "group:project-chromium-mac-arm64-tests-access"
+  }
+}
+realms {
+  name: "pools/try"
+  bindings {
+    role: "role/swarming.poolUser"
+    principals: "group:chromium-led-users"
+    principals: "project:chromium-m85"
+    principals: "project:chromium-m86"
+    principals: "user:infra-try-recipes-tester@chops-service-accounts.iam.gserviceaccount.com"
+  }
+}
 realms {
   name: "public"
   bindings {
@@ -162,6 +241,13 @@ realms {
     principals: "user:findit-for-me@appspot.gserviceaccount.com"
     principals: "user:tricium-prod@appspot.gserviceaccount.com"
   }
+  bindings {
+    role: "role/swarming.taskTriggerer"
+    principals: "group:chromium-led-users"
+    principals: "group:project-chromium-mac-arm64-tests-access"
+    principals: "group:project-chromium-tryjob-access"
+    principals: "user:infra-try-recipes-tester@chops-service-accounts.iam.gserviceaccount.com"
+  }
 }
 realms {
   name: "try-m85"

infra/config/lib/swarming.star

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Library for defining Swarming resources."""
+
+load("//lib/branches.star", "branches")
+
+def root_permissions():
+    """Sets up permissions that apply to all Chromium pools.
+
+    Noop on a non-main branch, since Swarming pools are owned by the primary
+    Chromium project defined on the main branch.
+    """
+    if not branches.matches(branches.MAIN_ONLY):
+        return
+
+    # Allow admins to cancel any task, delete bots, etc. in any Chromium pool.
+    luci.binding(
+        realm = "@root",
+        roles = "role/swarming.poolOwner",
+        groups = "project-chromium-admins",
+    )
+
+    # Allow everyone to see all tasks and bots in Chromium pools.
+    luci.binding(
+        realm = "@root",
+        roles = "role/swarming.poolViewer",
+        groups = "all",
+    )
+
+def pool_realm(*, name, groups = None, users = None, projects = None):
+    """Declares a realm with permissions for a Swarming pool.
+
+    `groups`, `users` and `projects` define who has "swarming.poolUser" role
+    which is required to submit tasks into the pool.
+
+    Individual Swarming pools are assigned to this realm in pools.cfg in
+    Swarming server-side configs.
+
+    Pools are owned by the main Chromium project and it makes sense to defined
+    them only on the main branch. This declaration is noop on a non-main branch.
+    """
+    if not branches.matches(branches.MAIN_ONLY):
+        return
+    if not name.startswith("pools/"):
+        fail("By convention Swarming pool realm name should start with pools/")
+
+    luci.realm(
+        name = name,
+        bindings = [
+            luci.binding(
+                roles = "role/swarming.poolUser",
+                groups = groups,
+                users = users,
+                projects = projects,
+            ),
+        ],
+    )
+
+def task_accounts(*, realm, groups = None, users = None):
+    """Declares what service accounts tasks in a realm can run as.
+
+    Used to declare accounts for isolated tests. There's no need to separately
+    declare accounts for Buildbucket builders since luci.builder(...) takes care
+    of that itself.
+    """
+    luci.binding(
+        realm = realm,
+        roles = "role/swarming.taskServiceAccount",
+        groups = groups,
+        users = users,
+    )
+
+def task_triggerers(*, builder_realm, pool_realm, users = None, groups = None):
+    """Declares who can launch arbitrary tasks.
+
+    Used to allow end users to launch LUCI Editor (aka LED) tasks and isolated
+    tests from their workstations.
+
+    The given users will be allowed to submit tasks in the `builder_realm` realm
+    (e.g. tasks that pretend to be "chromium/try" tasks), running on a Swarming
+    pool in some `pool_realm` (e.g. "pools/try" or "pools/tests").
+
+    Pools are owned by the main Chromium project, thus `pool_realm` setting is
+    effective only on the main branch where pool realms are defined. It is
+    silently skipped on on a non-main branch. Per-milestone projects still have
+    builders, so `builder_realm` setting is always effective.
+    """
+
+    # Permission to submit tasks to Swarming at all.
+    if branches.matches(branches.MAIN_ONLY):
+        luci.binding(
+            realm = pool_realm,
+            roles = "role/swarming.poolUser",
+            users = users,
+            groups = groups,
+        )
+
+    # Permission to associated tasks with the builder realm.
+    luci.binding(
+        realm = builder_realm,
+        roles = "role/swarming.taskTriggerer",
+        users = users,
+        groups = groups,
+    )
+
+swarming = struct(
+    root_permissions = root_permissions,
+    pool_realm = pool_realm,
+    task_accounts = task_accounts,
+    task_triggerers = task_triggerers,
+)

infra/config/main.star

@@ -117,6 +117,8 @@ luci.realm(
     ],
 )
 
+exec("//swarming.star")
+
 exec("//recipes.star")
 
 exec("//notifiers.star")

infra/config/project.star

@@ -41,12 +41,12 @@ def _generate_project_pyl(ctx):
 
 lucicfg.generator(_generate_project_pyl)
 
-# The branch numbers of branches that we have builders running for (including
-# milestone-specific projects)
-# Branch numbers for milestones can be viewed in the chromium column at
+# The milestone names and branch numbers of branches that we have builders
+# running for (including milestone-specific projects)
+# Branch numbers and milestones can be viewed in the chromium column at
 # https://chromiumdash.appspot.com/branches
 # The 2 highest milestones will be the ones with active branches
-ACTIVE_BRANCH_NUMBERS = [
-    4183,
-    4240,
+ACTIVE_BRANCHES = [
+    ("m85", 4183),
+    ("m86", 4240),
 ]

infra/config/subprojects/chromium/fallback-cq.star

@@ -2,7 +2,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-load("//project.star", "ACTIVE_BRANCH_NUMBERS")
+load("//project.star", "ACTIVE_BRANCHES")
 
 def _trailing_digit_regex(n):
     if n == 0:
@@ -88,9 +88,9 @@ def _get_fallback_branch_number_regexes():
     # Check that they're int first since we're relying on numeric qualities to
     # create the regexes
     branch_strs = []
-    for b in sorted(ACTIVE_BRANCH_NUMBERS):
+    for b in sorted([b for _, b in ACTIVE_BRANCHES]):
         if type(b) != type(0):
-            fail("The elements of ACTIVE_BRANCH_NUMBERS are expected to be ints," +
+            fail("Branch numbers in ACTIVE_BRANCHES are expected to be ints," +
                  " got {} ({})".format(type(b), b))
         branch_strs.append(str(b))
 

infra/config/subprojects/findit/findit.star

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 load("//lib/builders.star", "builder", "defaults", "goma", "os")
+load("//lib/swarming.star", swarming_lib = "swarming")
 
 luci.bucket(
     name = "findit",
@@ -20,6 +21,16 @@ luci.bucket(
     ],
 )
 
+# FindIt builders use a separate pool with a dedicated set of permissions.
+swarming_lib.pool_realm(name = "pools/findit")
+
+# Allow FindIt admins to run tasks directly to debug issues.
+swarming_lib.task_triggerers(
+    builder_realm = "findit",
+    pool_realm = "pools/findit",
+    groups = ["project-findit-owners"],
+)
+
 defaults.auto_builder_dimension.set(False)
 defaults.bucket.set("findit")
 defaults.build_numbers.set(True)

infra/config/swarming.star

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Permissions for Chromium main swarming pools (CI, try, tests).
+
+They are actually shared with a bunch other projects.
+"""
+
+load("//lib/swarming.star", "swarming")
+load("//project.star", "ACTIVE_BRANCHES")
+
+# Set up permissions that apply to all Chromium pools.
+swarming.root_permissions()
+
+# Task accounts for isolated tests.
+#
+# For simplicity of configuration we allow *any* task in the project (in any
+# realm) to run as any of these accounts. This is fine since all CI and Try
+# builders trigger isolated tasks in an identical way, using identical accounts
+# for isolated tests anyway.
+#
+# Note that this is declared on all branches, since task accounts "live" in a
+# project they are defined in, so we need to declare them for per-milestone
+# projects as well.
+swarming.task_accounts(
+    realm = "@root",  # i.e. inherit by all realms
+    groups = [
+        "project-chromium-test-task-accounts",
+    ],
+    users = [
+        # TODO(crbug.com/793982): Migrate uses of this account to a dedicated
+        # public test task account that's part of the group above, then delete
+        # this.
+        "ios-isolated-tester@chops-service-accounts.iam.gserviceaccount.com",
+    ],
+)
+
+# LED users that can trigger tasks in *any* realm in *any* pool.
+#
+# This should be used relatively sparingly. Prefer to configure the permissions
+# more precisely. E.g. see "chromium-led-users" below.
+swarming.task_triggerers(
+    builder_realm = "@root",
+    pool_realm = "@root",
+    groups = [
+        "mdb/chrome-troopers",
+        "google/luci-task-force@google.com",
+    ],
+)
+
+# Realm with bots that run CI builds (aka main waterfall bots).
+#
+# The tasks here are triggered via Buildbucket (which authenticates as
+# "project:<project that defines the bucket>"), so we enumerate projects
+# (besides "project:chromium" itself) that are allowed to use Chromium CI pools
+# in their Buildbucket configs (which are currently only per-milestone Chromium
+# projects).
+swarming.pool_realm(
+    name = "pools/ci",
+    projects = ["chromium-%s" % m for m, _ in ACTIVE_BRANCHES],
+)
+
+# Realm with bots that run try builds.
+#
+# The tasks here are also triggered via Buildbucket. See comment above.
+swarming.pool_realm(
+    name = "pools/try",
+    projects = ["chromium-%s" % m for m, _ in ACTIVE_BRANCHES],
+)
+
+# LED users that can trigger try builds via LED.
+swarming.task_triggerers(
+    builder_realm = "try",
+    pool_realm = "pools/try",
+    groups = [
+        "chromium-led-users",
+    ],
+    users = [
+        # An account used by "Build Recipes Tester" builder infra/try bucket
+        # used to tests changes to Chromium recipes using LED before commit.
+        "infra-try-recipes-tester@chops-service-accounts.iam.gserviceaccount.com",
+    ],
+)
+
+# Realm with bots that run isolated tests.
+#
+# Tasks here are triggered directly on Swarming (not via Buildbucket) by various
+# CI and Try builder (not only Chromium ones!) and also directly by users.
+swarming.pool_realm(
+    name = "pools/tests",
+    groups = [
+        # Various Chromium CI and Try LUCI builders that trigger isolated tests.
+        "project-chromium-ci-task-accounts",
+        "project-chromium-findit-task-accounts",
+        "project-chromium-try-task-accounts",
+
+        # DevTools uses Chrome pools for Layout tests.
+        "project-devtools-frontend-ci-task-accounts",
+        "project-devtools-frontend-try-task-accounts",
+
+        # V8 are reusing Chrome pools for isolated tests too.
+        "project-v8-ci-task-accounts",
+        "project-v8-try-task-accounts",
+
+        # ... and WebRTC.
+        "project-webrtc-ci-task-accounts",
+        "project-webrtc-try-task-accounts",
+
+        # Used by Pinpoint to trigger bisect jobs on machines in the Chrome-GPU pool.
+        "service-account-chromeperf",
+    ],
+    users = [
+        # Skia uses this pool directly.
+        "skia-external-ct-skps@skia-swarming-bots.iam.gserviceaccount.com",
+        # TODO(borenet): Remove the below after we're fully switched to Kitchen.
+        "chromium-swarm-bots@skia-swarming-bots.iam.gserviceaccount.com",
+    ],
+)
+
+# Anyone with Chromium tryjob access can use isolate testers pool directly.
+#
+# We assume isolated tests triggered from workstation go to the "try" realm,
+# just like tasks triggered by try jobs.
+swarming.task_triggerers(
+    builder_realm = "try",
+    pool_realm = "pools/tests",
+    groups = ["project-chromium-tryjob-access"],
+)
+
+# A separate realm for mac-arm64 bots, since they have different permissions.
+swarming.pool_realm(
+    name = "pools/tests-mac-arm64",
+    groups = [
+        # Allow CI builders (mac-arm64-rel-tests) to trigger tests.
+        "project-chromium-ci-task-accounts",
+    ],
+)
+
+# Users that can trigger mac-arm64 tasks.
+swarming.task_triggerers(
+    builder_realm = "try",
+    pool_realm = "pools/tests-mac-arm64",
+    groups = [
+        # Allowlist of people working on the mac-arm64 project. Contact
+        # srinivassista@ for access.
+        "project-chromium-mac-arm64-tests-access",
+    ],
+)