[dirmd] Add metadata-exporter builder

metadata-exporter builder is modernization of
https://ci.chromium.org/p/chromium/builders/ci/linux_chromium_component_updater

It uses a new recipe which uses a new tool that understands both
legacy OWNERS and new DIR_METADATA files.

Try using new bbagent (as opposed to kitchen) from the start.

Rename "component-mapping" notifier to "metadata-mapping" because it
is cheap, compared to changing mailing list emails and service
account emails.

Create a new Milo console for new builder.
Move existing builder from goma-specific console, because it has
nothing to do with goma.

Bug: 1102997
Change-Id: Ia0a988768c334b319fc11c38804c524a899fb119
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2300602
Commit-Queue: Nodir Turakulov <nodir@chromium.org>
Reviewed-by: Garrett Beaty <gbeaty@chromium.org>
Cr-Commit-Position: refs/heads/master@{#788752}

infra/config/generated/cr-buildbucket.cfg

@@ -11336,6 +11336,35 @@ buckets {
         }
       }
     }
+    builders {
+      name: "metadata_exporter"
+      swarming_host: "chromium-swarm.appspot.com"
+      swarming_tags: "vpython:native-python-wrapper"
+      dimensions: "builderless:1"
+      dimensions: "cores:8"
+      dimensions: "cpu:x86-64"
+      dimensions: "os:Ubuntu-16.04"
+      dimensions: "pool:luci.chromium.ci"
+      dimensions: "ssd:0"
+      exe {
+        cipd_package: "infra/recipe_bundles/chromium.googlesource.com/chromium/tools/build"
+        cipd_version: "refs/heads/master"
+        cmd: "luciexe"
+      }
+      properties: "{\"$build/goma\":{\"enable_ats\":true,\"jobs\":500,\"rpc_extra_params\":\"?prod\",\"server_host\":\"goma.chromium.org\"},\"$kitchen\":{\"devshell\":true,\"git_auth\":true},\"mastername\":\"chromium.linux\",\"recipe\":\"chromium_export_metadata\"}"
+      execution_timeout_secs: 10800
+      build_numbers: YES
+      service_account: "component-mapping-updater@chops-service-accounts.iam.gserviceaccount.com"
+      resultdb {
+        enable: true
+        bq_exports {
+          project: "luci-resultdb"
+          dataset: "chromium"
+          table: "ci_test_results"
+          test_results {}
+        }
+      }
+    }
     builders {
       name: "win-annotator-rel"
       swarming_host: "chromium-swarm.appspot.com"

infra/config/generated/luci-milo.cfg

@@ -9700,11 +9700,6 @@ consoles {
     category: "week3a|linux"
     short_name: "anno"
   }
-  builders {
-    name: "buildbucket/luci.chromium.ci/linux_chromium_component_updater"
-    category: "week3a|linux"
-    short_name: "cc_upd"
-  }
   builders {
     name: "buildbucket/luci.chromium.ci/linux-code-coverage"
     category: "week3a|linux"
@@ -15473,6 +15468,19 @@ consoles {
   }
   builder_view_only: true
 }
+consoles {
+  id: "metadata.exporter"
+  name: "metadata.exporter"
+  repo_url: "https://chromium.googlesource.com/chromium/src"
+  refs: "regexp:refs/heads/master"
+  manifest_name: "REVISION"
+  builders {
+    name: "buildbucket/luci.chromium.ci/linux_chromium_component_updater"
+  }
+  builders {
+    name: "buildbucket/luci.chromium.ci/metadata_exporter"
+  }
+}
 consoles {
   id: "tryserver.blink"
   name: "tryserver.blink"

infra/config/generated/luci-notify.cfg

@@ -2941,6 +2941,36 @@ notifiers {
     failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      rotation_urls: "https://chrome-ops-rotation-proxy.appspot.com/current/oncallator:chrome-build-sheriff"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "thomasanderson@chromium.org"
+    }
+  }
+  notifications {
+    on_new_status: FAILURE
+    email {
+      recipients: "chromium-component-mapping@google.com"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "metadata_exporter"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_new_status: FAILURE

infra/config/generated/luci-scheduler.cfg

@@ -11139,6 +11139,16 @@ job {
     builder: "mac-upload-perfetto"
   }
 }
+job {
+  id: "metadata_exporter"
+  schedule: "0 0,6,12,18 * * *"
+  acl_sets: "ci"
+  buildbucket {
+    server: "cr-buildbucket.appspot.com"
+    bucket: "luci.chromium.ci"
+    builder: "metadata_exporter"
+  }
+}
 job {
   id: "win-annotator-rel"
   acl_sets: "ci"

infra/config/notifiers.star

@@ -35,7 +35,7 @@ luci.notifier(
 )
 
 luci.notifier(
-    name = 'component-mapping',
+    name = 'metadata-mapping',
     on_new_status = ['FAILURE'],
     notify_emails = ['chromium-component-mapping@google.com'],
 )

infra/config/recipes.star

@@ -71,6 +71,11 @@ build_recipe(
     use_bbagent = True,
 )
 
+build_recipe(
+    name = 'recipe:chromium_export_metadata',
+    use_bbagent = True,
+)
+
 build_recipe(
     name = 'recipe:chromium_libfuzzer',
 )

infra/config/subprojects/chromium/master-only/ci.star

@@ -2899,15 +2899,25 @@ ci.linux_builder(
     os = os.LINUX_TRUSTY,
 )
 
+# TODO(crbug.com/1102997): remove this in favor of new "metadata_exporter"
+# builder.
 ci.linux_builder(
     name = 'linux_chromium_component_updater',
     executable = 'recipe:findit/chromium/update_components',
     schedule = '0 0,6,12,18 * * *',
     service_account = 'component-mapping-updater@chops-service-accounts.iam.gserviceaccount.com',
     triggered_by = [],
-    extra_notifies = ['component-mapping'],
+    extra_notifies = ['metadata-mapping'],
 )
 
+ci.linux_builder(
+    name = 'metadata_exporter',
+    executable = 'recipe:chromium_export_metadata',
+    schedule = '0 0,6,12,18 * * *',
+    service_account = 'component-mapping-updater@chops-service-accounts.iam.gserviceaccount.com',
+    triggered_by = [],
+    extra_notifies = ['metadata-mapping'],
+)
 
 ci.mac_ios_builder(
     name = 'ios-device',

infra/config/subprojects/chromium/master-only/consoles/metadata.exporter.star

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+luci.console_view(
+    name = 'metadata.exporter',
+    repo = 'https://chromium.googlesource.com/chromium/src',
+    entries = [
+        # TODO(crbug.com/1102997): remove this in favor of new
+        # "metadata_exporter" builder.
+        luci.console_view_entry(
+            builder = 'ci/linux_chromium_component_updater',
+        ),
+        luci.console_view_entry(
+            builder = 'ci/metadata_exporter',
+        ),
+    ],
+)

infra/config/subprojects/chromium/master-only/main.star

@@ -9,6 +9,7 @@ exec('./try.star')
 
 exec('./consoles/android.packager.star')
 exec('./consoles/luci.chromium.try.star')
+exec('./consoles/metadata.exporter.star')
 exec('./consoles/sheriff.ios.star')
 
 exec('./fallback-cq.star')

infra/config/subprojects/goma/consoles/chromium.goma.migration.star

@@ -374,11 +374,6 @@ luci.console_view(
             category = 'week3a|linux',
             short_name = 'anno',
         ),
-        luci.console_view_entry(
-            builder = 'ci/linux_chromium_component_updater',
-            category = 'week3a|linux',
-            short_name = 'cc_upd',
-        ),
         luci.console_view_entry(
             builder = 'ci/linux-code-coverage',
             category = 'week3a|linux',