Support tree closing in the chromium builders, add GPU closers.

Bug: 1094768
Change-Id: I8040777617c66483d4611e8791b018a3fdf3bee1
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2275358
Commit-Queue: Takuto Ikuta <tikuta@chromium.org>
Auto-Submit: Owen Rodley <orodley@chromium.org>
Reviewed-by: Takuto Ikuta <tikuta@chromium.org>
Cr-Commit-Position: refs/heads/master@{#783807}

infra/config/generated/luci-notify.cfg

@@ -95,6 +95,31 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "Android Release (Nexus 5X)"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -134,6 +159,81 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "GPU Linux Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "GPU Mac Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "GPU Win x64 Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -303,6 +403,30 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "Linux Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -355,6 +479,54 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "Mac Release (Intel)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "Mac Retina Release (AMD)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -550,6 +722,30 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci"
+    name: "Win10 x64 Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -793,6 +989,31 @@ notifiers {
     name: "linux_chromium_component_updater"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "Android Release (Nexus 5X)"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -819,6 +1040,177 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "GPU Linux Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "GPU Mac Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "GPU Win x64 Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "Linux Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "Mac Release (Intel)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "Mac Retina Release (AMD)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m83"
+    name: "Win10 x64 Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -908,6 +1300,31 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "Android Release (Nexus 5X)"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -934,6 +1351,177 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "GPU Linux Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "GPU Mac Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "GPU Win x64 Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "Linux Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "Mac Release (Intel)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "Mac Retina Release (AMD)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m84"
+    name: "Win10 x64 Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -1023,6 +1611,31 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "Android Release (Nexus 5X)"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true
@@ -1049,6 +1662,177 @@ notifiers {
     repository: "https://chromium.googlesource.com/chromium/src"
   }
 }
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "GPU Linux Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "GPU Mac Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "GPU Win x64 Builder"
+    repository: "https://chromium.googlesource.com/chromium/src"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "Linux Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "Mac Release (Intel)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "Mac Retina Release (AMD)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
+notifiers {
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+test-tree-closing-notifier@chromium.org"
+    }
+  }
+  notifications {
+    on_occurrence: FAILURE
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+    email {
+      recipients: "orodley+gpu-test-tree-closing-notifier@chromium.org"
+    }
+  }
+  builders {
+    bucket: "ci-m85"
+    name: "Win10 x64 Release (NVIDIA)"
+  }
+  tree_closers {
+    tree_status_host: "chromium-status.appspot.com"
+    failed_step_regexp: "bot_update|compile|gclient runhooks|runhooks|update"
+  }
+}
 notifiers {
   notifications {
     on_change: true

infra/config/lib/ci.star

@@ -421,6 +421,7 @@ def ci_builder(
     main_console_view=args.DEFAULT,
     cq_mirrors_console_view=args.DEFAULT,
     console_view_entry=None,
+    tree_closing=False,
     **kwargs):
   """Define a CI builder.
 
@@ -447,14 +448,21 @@ def ci_builder(
       has no effect on creating an entry to the main console view.
     console_view_entry - A structure providing the details of the entry
       to add to the console view. See `ci.console_view_entry` for details.
+    tree_closing - If true, failed builds from this builder that meet certain
+      criteria will close the tree and email the sheriff. See the
+      'chromium-tree-closer' config in notifiers.star for the full criteria.
   """
   # Define the builder first so that any validation of luci.builder arguments
   # (e.g. bucket) occurs before we try to use it
+  notifies = kwargs.pop('notifies', [])
+  if tree_closing:
+    notifies += ['chromium-tree-closer', 'chromium-tree-closer-email']
   ret = builders.builder(
       name = name,
       resultdb_bigquery_exports = [resultdb.export_test_results(
           bq_table = 'luci-resultdb.chromium.ci_test_results',
       )],
+      notifies = notifies,
       **kwargs
   )
 
@@ -766,22 +774,27 @@ def gpu_fyi_windows_builder(*, name, **kwargs):
   )
 
 
-def gpu_builder(*, name, **kwargs):
+def gpu_builder(*, name, tree_closing=True, **kwargs):
+  notifies = kwargs.pop('notifies', [])
+  if tree_closing:
+    notifies.append('gpu-tree-closer-email')
   return ci.builder(
       name = name,
       goma_backend = builders.goma.backend.RBE_PROD,
       mastername = 'chromium.gpu',
+      tree_closing = tree_closing,
+      notifies = notifies,
       **kwargs
   )
 
-
 # Many of the GPU testers are thin testers, they use linux VMS regardless of the
 # actual OS that the tests are built for
-def gpu_thin_tester(*, name, **kwargs):
+def gpu_thin_tester(*, name, tree_closing=True, **kwargs):
   return gpu_builder(
       name = name,
       cores = 2,
       os = builders.os.LINUX_DEFAULT,
+      tree_closing = tree_closing,
       **kwargs
   )
 

infra/config/main.star

@@ -16,6 +16,10 @@ lucicfg.check_version(
 # Enable LUCI Realms support.
 lucicfg.enable_experiment('crbug.com/1085650')
 
+# Enable tree closing. Note that for now it is only in a "dry run" mode, where
+# it doesn't actually close the tree, just logs what it would have done.
+lucicfg.enable_experiment("crbug.com/1054172")
+
 # Tell lucicfg what files it is allowed to touch
 lucicfg.config(
     config_dir = 'generated',

infra/config/notifiers.star

@@ -39,3 +39,34 @@ luci.notifier(
     on_new_status = ['FAILURE'],
     notify_emails = ['chromium-component-mapping@google.com'],
 )
+
+TREE_CLOSING_STEPS = [
+    'bot_update',
+    'compile',
+    'gclient runhooks',
+    'runhooks',
+    'update',
+]
+
+luci.tree_closer(
+    name = 'chromium-tree-closer',
+    tree_status_host = 'chromium-status.appspot.com',
+    failed_step_regexp = TREE_CLOSING_STEPS
+)
+
+luci.notifier(
+    name = 'chromium-tree-closer-email',
+    on_occurrence = ['FAILURE'],
+    # Stand-in for the chromium build sheriff, while testing.
+    notify_emails = ['orodley+test-tree-closing-notifier@chromium.org'],
+    failed_step_regexp = TREE_CLOSING_STEPS,
+)
+
+luci.notifier(
+    name = 'gpu-tree-closer-email',
+    on_occurrence = ['FAILURE'],
+    # Stand-in for chrome-gpu-build-failures@google.com and the GPU build
+    # sheriff, while testing.
+    notify_emails = ['orodley+gpu-test-tree-closing-notifier@chromium.org'],
+    failed_step_regexp = TREE_CLOSING_STEPS,
+)

infra/config/subprojects/chromium/master-only/ci.star

@@ -1935,6 +1935,7 @@ ci.gpu_builder(
     console_view_entry = ci.console_view_entry(
         category = 'Linux',
     ),
+    tree_closing = False,
 )
 
 ci.gpu_builder(
@@ -1944,6 +1945,7 @@ ci.gpu_builder(
     ),
     cores = None,
     os = os.MAC_ANY,
+    tree_closing = False,
 )
 
 ci.gpu_builder(
@@ -1953,6 +1955,7 @@ ci.gpu_builder(
         category = 'Windows',
     ),
     os = os.WINDOWS_ANY,
+    tree_closing = False,
 )
 
 
@@ -1962,6 +1965,7 @@ ci.gpu_thin_tester(
         category = 'Linux',
     ),
     triggered_by = ['GPU Linux Builder (dbg)'],
+    tree_closing = False,
 )
 
 ci.gpu_thin_tester(
@@ -1970,6 +1974,7 @@ ci.gpu_thin_tester(
         category = 'Mac',
     ),
     triggered_by = ['GPU Mac Builder (dbg)'],
+    tree_closing = False,
 )
 
 ci.gpu_thin_tester(
@@ -1978,6 +1983,7 @@ ci.gpu_thin_tester(
         category = 'Mac',
     ),
     triggered_by = ['GPU Mac Builder (dbg)'],
+    tree_closing = False,
 )
 
 ci.gpu_thin_tester(
@@ -1986,6 +1992,7 @@ ci.gpu_thin_tester(
         category = 'Windows',
     ),
     triggered_by = ['GPU Win x64 Builder (dbg)'],
+    tree_closing = False,
 )