mb: Add the ability to create expectation files for every builder.

Adds a `mb.py train` command that will dump json files for every
builder (one file for each master) to a specified dir. And hook up
`mb.py validate` to also validate the expectations.

This won't actually generate the expectations. For now it's silently
skipped if the expectations dir doesn't exist. Will create it in a
follow-up.

To see what the expectations would look like, see patchset 5:
https://chromium-review.googlesource.com/c/chromium/src/+/2378273/5

Bug: 1117577
Change-Id: Iaa7be9201d26fe331620f0d666a03a6adbaa2f3a
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2378273
Commit-Queue: Ben Pastene <bpastene@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Reviewed-by: Garrett Beaty <gbeaty@chromium.org>
Cr-Commit-Position: refs/heads/master@{#813417}

tools/mb/lib/validation.py

@@ -6,6 +6,7 @@
 import ast
 import collections
 import json
+import os
 import re
 
 
@@ -136,3 +137,24 @@ def CheckDuplicateConfigs(errs, config_pool, mixin_pool, grouping,
           'following configs are all equivalent: %s. Please '
           'consolidate these configs into only one unique name per '
           'configuration value.' % (', '.join(sorted('%r' % val for val in v))))
+
+
+def CheckExpectations(mbw, jsonish_blob, expectations_dir):
+  """Checks that the expectation files match the config file.
+
+  Returns: True if expectations are up-to-date. False otherwise.
+  """
+  # Assert number of masters == number of expectation files.
+  if len(mbw.ListDir(expectations_dir)) != len(jsonish_blob):
+    return False
+  for master, builders in jsonish_blob.items():
+    if not mbw.Exists(os.path.join(expectations_dir, master + '.json')):
+      return False  # No expecation file for the master.
+    expectation = mbw.ReadFile(os.path.join(expectations_dir, master + '.json'))
+    builders_json = json.dumps(builders,
+                               indent=2,
+                               sort_keys=True,
+                               separators=(',', ': '))
+    if builders_json != expectation:
+      return False  # Builders' expectation out of sync.
+  return True

tools/mb/mb.py

@@ -85,6 +85,8 @@ class MetaBuildWrapper(object):
     self.chromium_src_dir = CHROMIUM_SRC_DIR
     self.default_config = os.path.join(self.chromium_src_dir, 'tools', 'mb',
                                        'mb_config.pyl')
+    self.default_expectations = os.path.join(self.chromium_src_dir, 'tools',
+                                             'mb', 'mb_config_expectations')
     self.default_isolate_map = os.path.join(self.chromium_src_dir, 'testing',
                                             'buildbot', 'gn_isolate_map.pyl')
     self.executable = sys.executable
@@ -204,6 +206,28 @@ class MetaBuildWrapper(object):
                       help='path to goma directory')
     subp.set_defaults(func=self.CmdExport)
 
+    subp = subps.add_parser('train',
+                            description='Writes the expanded configuration '
+                            'for each builder as JSON files to a configured '
+                            'directory.')
+    subp.add_argument('-f',
+                      '--config-file',
+                      metavar='PATH',
+                      help='path to config file (default is mb_config.pyl')
+    subp.add_argument('--expectations-dir',
+                      metavar='PATH',
+                      help='path to dir containing expectation files')
+    subp.add_argument('-n',
+                      '--dryrun',
+                      action='store_true',
+                      help='Do a dry run (i.e., do nothing, just print '
+                      'the commands that will run)')
+    subp.add_argument('-v',
+                      '--verbose',
+                      action='store_true',
+                      help='verbose logging')
+    subp.set_defaults(func=self.CmdTrain)
+
     subp = subps.add_parser('gen',
                             description='Generate a new set of build files.')
     AddCommonOptions(subp)
@@ -316,6 +340,9 @@ class MetaBuildWrapper(object):
                             description='Validate the config file.')
     subp.add_argument('-f', '--config-file', metavar='PATH',
                       help='path to config file (default is %(default)s)')
+    subp.add_argument('--expectations-dir',
+                      metavar='PATH',
+                      help='path to dir containing expectation files')
     subp.set_defaults(func=self.CmdValidate)
 
     subp = subps.add_parser('zip',
@@ -377,6 +404,25 @@ class MetaBuildWrapper(object):
     self.Print(s)
     return 0
 
+  def CmdTrain(self):
+    expectations_dir = self.args.expectations_dir or self.default_expectations
+    if not self.Exists(expectations_dir):
+      self.Print('Expectations dir (%s) does not exist.' % expectations_dir)
+      return 1
+    # Removing every expectation file then immediately re-generating them will
+    # clear out deleted groups.
+    for f in self.ListDir(expectations_dir):
+      self.RemoveFile(os.path.join(expectations_dir, f))
+    obj = self._ToJsonish()
+    for master, builder in sorted(obj.items()):
+      expectation_file = os.path.join(expectations_dir, master + '.json')
+      json_s = json.dumps(builder,
+                          indent=2,
+                          sort_keys=True,
+                          separators=(',', ': '))
+      self.WriteFile(expectation_file, json_s)
+    return 0
+
   def CmdGen(self):
     vals = self.Lookup()
     return self.RunGNGen(vals)
@@ -723,6 +769,14 @@ class MetaBuildWrapper(object):
                    (self.args.config_file if self.args.config_file else self.
                     default_config)) + '\n  ' + '\n  '.join(errs))
 
+    expectations_dir = self.args.expectations_dir or self.default_expectations
+    # TODO(crbug.com/1117577): Force all versions of mb_config.pyl to have
+    # expectations. For now, just ignore those that don't have them.
+    if self.Exists(expectations_dir):
+      jsonish_blob = self._ToJsonish()
+      if not validation.CheckExpectations(self, jsonish_blob, expectations_dir):
+        raise MBErr("Expectations out of date. Please run 'mb.py train'.")
+
     if print_ok:
       self.Print('mb config file %s looks ok.' %
                  (self.args.config_file
@@ -1798,6 +1852,10 @@ class MetaBuildWrapper(object):
     f.close()
     return contents
 
+  def ListDir(self, path):
+    # This function largely exists so it can be overridden for testing.
+    return os.listdir(path)
+
   def MaybeMakeDirectory(self, path):
     try:
       os.makedirs(path)

tools/mb/mb_unittest.py

@@ -12,6 +12,7 @@ import json
 import os
 import re
 import sys
+import tempfile
 import unittest
 
 if sys.version_info.major == 2:
@@ -52,6 +53,7 @@ class FakeMBW(mb.MetaBuildWrapper):
       self.cwd = '/fake_src/out/Default'
 
     self.files = {}
+    self.dirs = set()
     self.calls = []
     self.cmds = []
     self.cross_compile = None
@@ -63,11 +65,20 @@ class FakeMBW(mb.MetaBuildWrapper):
     return '$HOME/%s' % path
 
   def Exists(self, path):
-    return self.files.get(self._AbsPath(path)) is not None
+    abs_path = self._AbsPath(path)
+    return (self.files.get(abs_path) is not None or abs_path in self.dirs)
+
+  def ListDir(self, path):
+    dir_contents = []
+    for f in list(self.files.keys()) + list(self.dirs):
+      head, _ = os.path.split(f)
+      if head == path:
+        dir_contents.append(f)
+    return dir_contents
 
   def MaybeMakeDirectory(self, path):
     abpath = self._AbsPath(path)
-    self.files[abpath] = True
+    self.dirs.add(abpath)
 
   def PathJoin(self, *comps):
     return self.sep.join(comps)
@@ -96,6 +107,11 @@ class FakeMBW(mb.MetaBuildWrapper):
     else:
       self.out += sep.join(args) + end
 
+  def TempDir(self):
+    tmp_dir = os.path.join(tempfile.gettempdir(), 'mb_test')
+    self.dirs.add(tmp_dir)
+    return tmp_dir
+
   def TempFile(self, mode='w'):
     return FakeFile(self.files)
 
@@ -786,6 +802,12 @@ class UnitTest(unittest.TestCase):
                     'enable_doom_melon = true\n'
                     'use_goma = true\n'))
 
+  def test_train(self):
+    mbw = self.fake_mbw()
+    temp_dir = mbw.TempDir()
+    self.check(['train', '--expectations-dir', temp_dir], mbw=mbw, ret=0)
+    self.assertIn(os.path.join(temp_dir, 'fake_master.json'), mbw.files)
+
   def test_validate(self):
     mbw = self.fake_mbw()
     self.check(['validate'], mbw=mbw, ret=0)
@@ -804,6 +826,25 @@ class UnitTest(unittest.TestCase):
         'following configs are all equivalent: \'some_config\', '
         '\'some_other_config\'.', mbw.out)
 
+  def test_good_expectations_validate(self):
+    mbw = self.fake_mbw()
+    # Train the expectations normally.
+    temp_dir = mbw.TempDir()
+    self.check(['train', '--expectations-dir', temp_dir], mbw=mbw, ret=0)
+    # Immediately validating them should pass.
+    self.check(['validate', '--expectations-dir', temp_dir], mbw=mbw, ret=0)
+
+  def test_bad_expectations_validate(self):
+    mbw = self.fake_mbw()
+    # Train the expectations normally.
+    temp_dir = mbw.TempDir()
+    self.check(['train', '--expectations-dir', temp_dir], mbw=mbw, ret=0)
+    # Remove one of the expectation files.
+    mbw.files.pop(os.path.join(temp_dir, 'fake_master.json'))
+    # Now validating should fail.
+    self.check(['validate', '--expectations-dir', temp_dir], mbw=mbw, ret=1)
+    self.assertIn('Expectations out of date', mbw.out)
+
   def test_build_command_unix(self):
     files = {
         '/fake_src/out/Default/toolchain.ninja':