Add a script for clobbering arbitrary caches.

named-cache-clobber.py can be used to clobber any named cache on
machines in any pool. The existing clobber.py was replaced with
builder-cache-clobber.py, which renames the pool flag to bucket
to more accurately indicate the expected value and removes the
restriction on project.

Change-Id: I7614330213c8e20a0c0e9ee4bb0498d3b2fbbb38
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1757888
Commit-Queue: Garrett Beaty <gbeaty@chromium.org>
Reviewed-by: John Budorick <jbudorick@chromium.org>
Cr-Commit-Position: refs/heads/master@{#688205}

tools/infra/builder-cache-clobber.py

+#!/usr/bin/env python
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Clobbers all builder caches for a specific builder.
+
+Note that this currently does not support windows.
+"""
+
+import argparse
+import hashlib
+import sys
+
+import clobber_cache_utils
+
+
+def main(raw_args):
+  parser = argparse.ArgumentParser()
+  clobber_cache_utils.add_common_args(parser)
+  parser.add_argument('--builder', required=True)
+  parser.add_argument('--bucket', required=True)
+  parser.add_argument('--project', default='chromium')
+  args = parser.parse_args(raw_args)
+
+  # Matches http://bit.ly/2WZO33P
+  h = hashlib.sha256('%s/%s/%s' % (args.project, args.bucket, args.builder))
+  cache = 'builder_%s_v2' % (h.hexdigest())
+  pool = 'luci.%s.%s' % (args.project, args.bucket)
+
+  clobber_cache_utils.clobber_caches(args.swarming_server, pool, cache,
+                                     'cache/builder', args.dry_run)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))

tools/infra/clobber.py → tools/infra/clobber_cache_utils.py

-#!/usr/bin/env python
 # Copyright 2019 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Clobbers all builder caches for a specific builder.
-
-Note that this currently does not support windows.
-"""
+"""Utility functions for cache clobbering scripts."""
 
 from __future__ import print_function
 
-import argparse
-import hashlib
 import os
 import subprocess
 import sys
@@ -22,14 +16,14 @@ _SWARMING_CLIENT = os.path.join(_SRC_ROOT, 'tools', 'swarming_client',
 _SWARMING_SERVER = 'chromium-swarm.appspot.com'
 
 
-def _get_bots(pool, cache):
+def _get_bots(swarming_server, pool, cache):
   cmd = [
       sys.executable,
       _SWARMING_CLIENT,
       'bots',
       '-b',
       '-S',
-      _SWARMING_SERVER,
+      swarming_server,
       '-d',
       'caches',
       cache,
@@ -40,13 +34,14 @@ def _get_bots(pool, cache):
   return subprocess.check_output(cmd).splitlines()
 
 
-def _trigger_clobber(pool, cache, bot, dry_run):
+def _trigger_clobber(swarming_server, pool, cache, bot, mount_rel_path,
+                     dry_run):
   cmd = [
       sys.executable,
       _SWARMING_CLIENT,
       'trigger',
       '-S',
-      _SWARMING_SERVER,
+      swarming_server,
       '-d',
       'pool',
       pool,
@@ -55,14 +50,14 @@ def _trigger_clobber(pool, cache, bot, dry_run):
       bot,
       '--named-cache',
       cache,
-      'cache/builder',
+      mount_rel_path,
       '--priority=10',
       '--raw-cmd',
       '--',
       # TODO(jbudorick): Generalize this for windows.
       '/bin/rm',
       '-rf',
-      'cache/builder',
+      mount_rel_path,
   ]
   if dry_run:
     print('Would run `%s`' % ' '.join(cmd))
@@ -70,21 +65,41 @@ def _trigger_clobber(pool, cache, bot, dry_run):
     subprocess.check_call(cmd)
 
 
-def main(raw_args):
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--builder', required=True)
-  parser.add_argument('--pool', required=True)
-  parser.add_argument(
-      '--project', default='chromium', choices=['chromium', 'infra'])
-  parser.add_argument('-n', '--dry-run', action='store_true')
-  args = parser.parse_args(raw_args)
-
-  # Matches http://bit.ly/2WZO33P
-  h = hashlib.sha256('%s/%s/%s' % (args.project, args.pool, args.builder))
-  cache_name = 'builder_%s_v2' % (h.hexdigest())
-  swarming_pool = 'luci.%s.%s' % (args.project, args.pool)
-
-  bots = _get_bots(swarming_pool, cache_name)
+def add_common_args(argument_parser):
+  """Add common arguments to the argument parser used for cache clobber scripts.
+
+  The following arguments will be added to the argument parser:
+    * swarming_server (-S/--swarming-server) - The swarming server instance to
+      lookup bots to clobber caches on, with a default of the
+      chromium-swarm.appspot.com.
+    * dry_run (-n/--dry-run) - Whether a dry-run should be performed rather than
+      actually clobbering caches, defaults to False.
+  """
+  argument_parser.add_argument(
+      '-S', '--swarming-server', default=_SWARMING_SERVER)
+  argument_parser.add_argument('-n', '--dry-run', action='store_true')
+
+
+def clobber_caches(swarming_server, pool, cache, mount_rel_path, dry_run):
+  """Clobber caches on bots.
+
+  The set of bots in `pool` in `swarming_server` with a cache named `cache` will
+  be looked up and printed out then the user will be asked to confirm that the
+  caches should be clobbered. If the user confirms, tasks that clobber the cache
+  will be triggered on each bot or if `dry_run` is true, the command that would
+  trigger such a task is printed instead.
+
+  Args:
+    * swarming_server - The swarming_server instance to lookup bots to clobber
+      caches on.
+    * pool - The pool of machines to lookup bots to clobber caches on.
+    * cache - The name of the cache to clobber.
+    * mount_rel_path - The relative path to mount the cache to when clobbering.
+    * dry_run - Whether a dry-run should be performed where the commands that
+      would be executed to trigger the clobber task are printed rather than
+      actually triggering the clobber task.
+  """
+  bots = _get_bots(swarming_server, pool, cache)
 
   print('The following bots will be clobbered:')
   print()
@@ -97,9 +112,4 @@ def main(raw_args):
     return 1
 
   for bot in bots:
-    _trigger_clobber(swarming_pool, cache_name, bot, args.dry_run)
-  return 0
-
-
-if __name__ == '__main__':
-  sys.exit(main(sys.argv[1:]))
+    _trigger_clobber(swarming_server, pool, cache, bot, mount_rel_path, dry_run)

tools/infra/named-cache-clobber.py

+#!/usr/bin/env python
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Clobbers all instances of a named cache.
+
+Note that this currently does not support windows.
+"""
+
+import argparse
+import sys
+
+import clobber_cache_utils
+
+
+def main(raw_args):
+  parser = argparse.ArgumentParser()
+  clobber_cache_utils.add_common_args(parser)
+  parser.add_argument('--pool', required=True)
+  parser.add_argument('--cache', required=True)
+  parser.add_argument('--mount-rel-path')
+  args = parser.parse_args(raw_args)
+
+  mount_rel_path = args.mount_rel_path
+  if mount_rel_path is None:
+    mount_rel_path = 'cache/%s' % args.cache
+
+  clobber_cache_utils.clobber_caches(args.swarming_server, args.pool,
+                                     args.cache, mount_rel_path, args.dry_run)
+
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv[1:]))