[GRIT] Add grit update_resource_ids for structure-preserving update of resource_ids files.

Previously resource_ids requires manual curation. These files contain
an implicit "structure" that imposes grouping, dependency (avoids
overlap), and parallel streams (allows overlap) of item lists.

It turns out that the implicit structure of start IDs can be modeled
as a Series-Parallel (SP) Graph. By introducing "META": {"join": #}
fields to resource_ids, the graph structure can be made concrete.

This CL adds a new GRIT tool that updates resource_ids file while
preserving structure. The inputs are:
* Existing start ID assignment, whose numerical sequence (along with
  newly added "join" meta data) stores the SP Graph.
* GRD files linked to by resource_ids, which are parsed to find per-tag
  ID usages.
  * For GRD files that may be unavailable (e.g., generated ones, or
    src-internal ones), "META": {"size": {...}} fields is needed to
    specify bounds for per-tag ID usage.

The main output is the resource_ids file with renumbered start IDs.
Debug flags are available to access intermediate data.

Chrome's main resource_ids file is updated to include the new "META"
flags. However, as a precaution, the start IDs are not yet reassigned
using the tool -- this will be done as a follow-up.

Bug: 979886

Change-Id: If7c0a43f57a1d27950784da7eb8405a6a0567f03
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1895736
Commit-Queue: Samuel Huang <huangs@chromium.org>
Reviewed-by: Andrew Grieve <agrieve@chromium.org>
Cr-Commit-Position: refs/heads/master@{#715838}

tools/grit/grit/grit_runner.py

@@ -66,6 +66,12 @@ def ToolFactoryUnit():
   import grit.tool.unit
   return grit.tool.unit.UnitTestTool()
 
+
+def ToolFactoryUpdateResourceIds():
+  import grit.tool.update_resource_ids
+  return grit.tool.update_resource_ids.UpdateResourceIds()
+
+
 def ToolFactoryXmb():
   import grit.tool.xmb
   return grit.tool.xmb.OutputXmb()
@@ -82,28 +88,73 @@ _HIDDEN = 3  # optional key - presence indicates tool is hidden
 # Maps tool names to the tool's module.  Done as a list of (key, value) tuples
 # instead of a map to preserve ordering.
 _TOOLS = [
-  ['android2grd', {
-      _FACTORY: ToolAndroid2Grd,
-      _REQUIRES_INPUT : False }],
-  ['build', { _FACTORY : ToolFactoryBuild, _REQUIRES_INPUT : True }],
-  ['buildinfo', { _FACTORY : ToolFactoryBuildInfo, _REQUIRES_INPUT : True }],
-  ['count', { _FACTORY : ToolFactoryCount, _REQUIRES_INPUT : True }],
-  ['menufromparts', {
-      _FACTORY: ToolFactoryMenuTranslationsFromParts,
-      _REQUIRES_INPUT : True, _HIDDEN : True }],
-  ['newgrd', { _FACTORY  : ToolFactoryNewGrd, _REQUIRES_INPUT : False }],
-  ['rc2grd', { _FACTORY : ToolFactoryRc2Grd, _REQUIRES_INPUT : False }],
-  ['resize', {
-      _FACTORY : ToolFactoryResizeDialog, _REQUIRES_INPUT : True }],
-  ['sdiff', { _FACTORY : ToolFactoryDiffStructures,
-              _REQUIRES_INPUT : False }],
-  ['test', {
-      _FACTORY: ToolFactoryTest, _REQUIRES_INPUT : True,
-      _HIDDEN : True }],
-  ['transl2tc', { _FACTORY : ToolFactoryTranslationToTc,
-                  _REQUIRES_INPUT : False }],
-  ['unit', { _FACTORY : ToolFactoryUnit, _REQUIRES_INPUT : False }],
-  ['xmb', { _FACTORY : ToolFactoryXmb, _REQUIRES_INPUT : True }],
+    ['android2grd', {
+        _FACTORY: ToolAndroid2Grd,
+        _REQUIRES_INPUT: False
+    }],
+    ['build', {
+        _FACTORY: ToolFactoryBuild,
+        _REQUIRES_INPUT: True
+    }],
+    ['buildinfo', {
+        _FACTORY: ToolFactoryBuildInfo,
+        _REQUIRES_INPUT: True
+    }],
+    ['count', {
+        _FACTORY: ToolFactoryCount,
+        _REQUIRES_INPUT: True
+    }],
+    [
+        'menufromparts',
+        {
+            _FACTORY: ToolFactoryMenuTranslationsFromParts,
+            _REQUIRES_INPUT: True,
+            _HIDDEN: True
+        }
+    ],
+    ['newgrd', {
+        _FACTORY: ToolFactoryNewGrd,
+        _REQUIRES_INPUT: False
+    }],
+    ['rc2grd', {
+        _FACTORY: ToolFactoryRc2Grd,
+        _REQUIRES_INPUT: False
+    }],
+    ['resize', {
+        _FACTORY: ToolFactoryResizeDialog,
+        _REQUIRES_INPUT: True
+    }],
+    ['sdiff', {
+        _FACTORY: ToolFactoryDiffStructures,
+        _REQUIRES_INPUT: False
+    }],
+    ['test', {
+        _FACTORY: ToolFactoryTest,
+        _REQUIRES_INPUT: True,
+        _HIDDEN: True
+    }],
+    [
+        'transl2tc',
+        {
+            _FACTORY: ToolFactoryTranslationToTc,
+            _REQUIRES_INPUT: False
+        }
+    ],
+    ['unit', {
+        _FACTORY: ToolFactoryUnit,
+        _REQUIRES_INPUT: False
+    }],
+    [
+        'update_resource_ids',
+        {
+            _FACTORY: ToolFactoryUpdateResourceIds,
+            _REQUIRES_INPUT: False
+        }
+    ],
+    ['xmb', {
+        _FACTORY: ToolFactoryXmb,
+        _REQUIRES_INPUT: True
+    }],
 ]
 
 

tools/grit/grit/node/misc.py

@@ -145,21 +145,21 @@ def _ComputeIds(root, predetermined_tids):
 
       elif ('offset' in item.attrs and group and
             group.attrs.get('first_id', '') != ''):
-         offset_text = item.attrs['offset']
-         parent_text = group.attrs['first_id']
+        offset_text = item.attrs['offset']
+        parent_text = group.attrs['first_id']
 
-         try:
-           offset_id = long(offset_text)
-         except ValueError:
-           offset_id = tids[offset_text]
+        try:
+          offset_id = long(offset_text)
+        except ValueError:
+          offset_id = tids[offset_text]
 
-         try:
-           parent_id = long(parent_text)
-         except ValueError:
-           parent_id = tids[parent_text]
+        try:
+          parent_id = long(parent_text)
+        except ValueError:
+          parent_id = tids[parent_text]
 
-         id = parent_id + offset_id
-         reason = 'first_id %d + offset %d' % (parent_id, offset_id)
+        id = parent_id + offset_id
+        reason = 'first_id %d + offset %d' % (parent_id, offset_id)
 
       # We try to allocate IDs sequentially for blocks of items that might
       # be related, for instance strings in a stringtable (as their IDs might be
@@ -513,7 +513,8 @@ class GritNode(base.Node):
     """Returns the distinct (language, context, fallback_to_default_layout)
     triples from the output nodes.
     """
-    return set((n.GetLanguage(), n.GetContext(), n.GetFallbackToDefaultLayout()) for n in self.GetOutputFiles())
+    return set((n.GetLanguage(), n.GetContext(), n.GetFallbackToDefaultLayout())
+               for n in self.GetOutputFiles())
 
   def GetSubstitutionMessages(self):
     """Returns the list of <message sub_variable="true"> nodes."""

tools/grit/grit/tool/update_resource_ids/__init__.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Package grit.tool.update_resource_ids
+
+Updates GRID resource_ids from linked GRD files, while preserving structure.
+
+A resource_ids file is a JSON dict (with Python comments) that maps GRD paths
+to *items*. Item order is ignored by GRIT, but is important since it establishes
+a narrative of item dependency (needs non-overlapping IDs) and mutual exclusion
+(allows ID overlap). Example:
+
+{
+  # The first entry in the file, SRCDIR, is special: It is a relative path from
+  # this file to the base of your checkout.
+  "SRCDIR": "../..",
+
+  # First GRD file. This entry is an "Item".
+  "1.grd": {
+    "messages": [400],  # "Tag".
+  },
+  # Depends on 1.grd, i.e., 500 >= 400 + (# of IDs used in 1.grd).
+  "2a.grd": {
+    "includes": [500],    # "Tag".
+    "structures": [510],  # "Tag" (etc.).
+  },
+  # Depends on 2a.grd.
+  "3a.grd": {
+    "includes": [1000],
+  },
+  # Depends on 2a.grd, but overlaps with 3b.grd due to mutually exclusivity.
+  "3b.grd": {
+    "includes": [1000],
+  },
+  # Depends on {3a.grd, 3b.grd}.
+  "4.grd": {
+    "META": {"join": 2},  # Hint for update_resource_ids.
+    "structures": [1500],
+  },
+  # Depends on 1.grd but overlaps with 2a.grd.
+  "2b.grd": {
+    "includes": [500],
+    "structures": [540],
+  },
+  # Depends on {4.grd, 2b.grd}.
+  "5.grd": {
+    "META": {"join": 2},  # Hint for update_resource_ids.
+    "includes": [600],
+  },
+  # Depends on 5.grd. File is generated, so hint is needed for sizes.
+  "<(SHARED_INTERMEDIATE_DIR)/6.grd": {
+    "META": {"sizes": {"includes": [10]}},
+    "includes": [700],
+  },
+}
+
+The "structure" within a resouces_ids file are as follows:
+1. Comments and spacing.
+2. Item ordering, to establish dependency and grouping.
+3. Special provision to allow ID overlaps from mutual exclusion.
+
+This module parses a resource_ids file, reads ID usages from GRD files it refers
+to, and generates an updated version of the resource_ids file while preserving
+structure elements 1-3 stated above.
+"""
+
+from __future__ import print_function
+
+import collections
+import getopt
+import os
+import sys
+
+from grit.tool import interface
+from grit.tool.update_resource_ids import assigner, common, parser, reader
+
+
+def _ReadData(input_file):
+  if input_file == '-':
+    data = sys.stdin.read()
+    file_dir = os.getcwd()
+  else:
+    with open(input_file, 'rt') as f:
+      data = f.read()
+    file_dir = os.path.dirname(input_file)
+  return data, file_dir
+
+
+def _MultiReplace(data, repl):
+  """Multi-replacement of text |data| by ranges and replacement text.
+
+  Args:
+    data: Original text.
+    repl: List of (lo, hi, s) tuples, specifying that |data[lo:hi]| should be
+      replaced with |s|. The ranges must be inside |data|, and not overlap.
+  Returns: New text.
+  """
+  res = []
+  prev = 0
+  for (lo, hi, s) in sorted(repl):
+    if prev < lo:
+      res.append(data[prev:lo])
+    res.append(s)
+    prev = hi
+  res.append(data[prev:])
+  return ''.join(res)
+
+
+def _CreateAndOutputResult(data, repl, output):
+  new_data = _MultiReplace(data, repl)
+  if output:
+    with open(output, 'wt') as fh:
+      fh.write(new_data)
+  else:
+    sys.stdout.write(new_data)
+
+
+class _Args:
+  """Encapsulated arguments for this module."""
+  def __init__(self):
+    self.input = None
+    self.output = None
+    self.count = False
+    self.fake = False
+    self.naive = False
+    self.parse = False
+    self.tokenize = False
+
+  @staticmethod
+  def Parse(raw_args):
+    own_opts, raw_args = getopt.getopt(raw_args, 'o:cpt', [
+        'count',
+        'fake',
+        'naive',
+        'parse',
+        'tokenize',
+    ])
+    args = _Args();
+    if not len(raw_args) == 1:
+      print('grit update_resource_ids takes exactly one argument, the path to '
+            'the resource ids file.')
+      return 2
+    args.input = raw_args[0]
+    for (key, val) in own_opts:
+      if key == '-o':
+        args.output = val
+      elif key in ('--count', '-c'):
+        args.count = True
+      elif key == '--fake':
+        args.fake = True
+      elif key == '--naive':
+        args.naive = True
+      elif key in ('--parse', '-p'):
+        args.parse = True
+      elif key in ('--tokenize', '-t'):
+        args.tokenize = True
+    return args
+
+
+class UpdateResourceIds(interface.Tool):
+  """Updates all starting IDs in an resource_ids file by reading all GRD files
+it refers to, estimating the number of required IDs of each type, then rewrites
+starting IDs while preserving structure.
+
+Usage: grit update_resource_ids [--parse|-p] [--read-grd|-r] [--tokenize|-t]
+                                [--naive] [--fake] [-o OUTPUT_FILE]
+                                RESOURCE_IDS_FILE
+
+RESOURCE_IDS_FILE is the path of the input resource_ids file.
+
+The -o option redirects output (default stdout) to OUPTUT_FILE, which can also
+be RESOURCE_IDS_FILE.
+
+Other options:
+
+  -E NAME=VALUE     Sets environment variable NAME to VALUE (within grit).
+
+  --count|-c        Parses RESOURCE_IDS_FILE, reads the GRD files, and prints
+                    required sizes.
+
+  --fake            For testing: Skips reading GRD files, and assigns 10 as the
+                    usage of every tag.
+
+  --naive           Use naive coarse assigner.
+
+  --parse|-p        Parses RESOURCE_IDS_FILE and dumps its nodes to console.
+
+  --tokenize|-t     Tokenizes RESOURCE_IDS_FILE and reprints it as syntax-
+                    highlighted output.
+
+"""
+
+  def __init(self):
+    super(UpdateResourceIds, self).__init__()
+
+  def ShortDescription(self):
+    return 'Updates a resource_ids file based on usage, preserving structure'
+
+  def _DumpTokens(self, data, tok_gen):
+    # Reprint |data| with syntax highlight.
+    color_map = {
+        '#': common.Color.GRAY,
+        'S': common.Color.CYAN,
+        '0': common.Color.RED,
+        '{': common.Color.YELLOW,
+        '}': common.Color.YELLOW,
+        '[': common.Color.GREEN,
+        ']': common.Color.GREEN,
+        ':': common.Color.MAGENTA,
+        ',': common.Color.MAGENTA,
+    }
+    for t, lo, hi in tok_gen:
+      c = color_map.get(t, common.Color.NONE)
+      sys.stdout.write(c(data[lo:hi]))
+
+  def _DumpRootObj(self, root_obj):
+    print(root_obj)
+
+  def _DumpResourceCounts(self, usage_gen):
+    tot = collections.Counter()
+    for item, tag_name_to_usage in usage_gen:
+      c = common.Color.YELLOW if item.grd.startswith('<') else common.Color.CYAN
+      print('%s: %r' % (c(item.grd), dict(tag_name_to_usage)))
+      tot += collections.Counter(tag_name_to_usage)
+    print(common.Color.GRAY('-' * 80))
+    print('%s: %r' % (common.Color.GREEN('Total'), dict(tot)))
+    print('%s: %d' % (common.Color.GREEN('Grand Total'), sum(tot.values())))
+
+  def Run(self, opts, raw_args):
+    self.SetOptions(opts)
+
+    args = _Args.Parse(raw_args)
+    data, file_dir = _ReadData(args.input)
+
+    tok_gen = parser.Tokenize(data)
+    if args.tokenize:
+      return self._DumpTokens(data, tok_gen)
+
+    root_obj = parser.ResourceIdParser(data, tok_gen).Parse()
+    if args.parse:
+      return self._DumpRootObj(root_obj)
+    item_list = common.BuildItemList(root_obj)
+
+    src_dir = os.path.abspath(os.sep.join([file_dir, root_obj['SRCDIR'].val]))
+    usage_gen = reader.GenerateResourceUsages(item_list, src_dir, args.fake)
+    if args.count:
+      return self._DumpResourceCounts(usage_gen)
+    for item, tag_name_to_usage in usage_gen:
+      item.SetUsages(tag_name_to_usage)
+
+    new_ids_gen = assigner.GenerateNewIds(item_list, args.naive)
+    # Create replacement specs usable by _MultiReplace().
+    repl = [(tag.lo, tag.hi, str(new_id)) for tag, new_id in new_ids_gen]
+    _CreateAndOutputResult(data, repl, args.output)

tools/grit/grit/tool/update_resource_ids/assigner_unittest.py

+#!/usr/bin/env python
+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from __future__ import print_function
+
+import os
+import sys
+import traceback
+import unittest
+if __name__ == '__main__':
+  sys.path.append(os.path.join(os.path.dirname(__file__), '../../..'))
+
+from grit.tool.update_resource_ids import assigner, common, parser
+
+# |spec| format: A comma-separated list of (old) starting IDs. Modifiers:
+# * Prefix with n '*' to assign the item's META "join" field to n + 1.
+# * Suffix with "+[usage]" to assign |usage| for the item (else default=10)
+
+
+def _RenderTestResourceId(spec):
+  """Renders barebone resource_ids data based on |spec|."""
+  data = '{"SRCDIR": ".",'
+  for i, tok in enumerate(spec.split(',')):
+    num_star = len(tok) - len(tok.lstrip('*'))
+    tok = tok[num_star:]
+    meta = '"META":{"join": %d},' % (num_star + 1) if num_star else ''
+    start_id = tok.split('+')[0]  # Strip '+usage'.
+    data += '"foo%d.grd": {%s"includes": [%s]},' % (i, meta, start_id)
+  data += '}'
+  return data
+
+
+def _CreateTestItemList(spec):
+  """Creates list of ItemInfo based on |spec|."""
+  data = _RenderTestResourceId(spec)
+  item_list = common.BuildItemList(
+      parser.ResourceIdParser(data, parser.Tokenize(data)).Parse())
+  # Assign usages from "id+usage", default to 10.
+  for i, tok in enumerate(spec.split(',')):
+    item_list[i].tags[0].usage = int((tok.split('+') + ['10'])[1])
+  return item_list
+
+
+def _RunCoarseIdAssigner(spec):
+  item_list = _CreateTestItemList(spec)
+  coarse_id_assigner = assigner.DagCoarseIdAssigner(item_list, 1)
+  new_id_list = []  # List of new IDs, to check ID assignment.
+  new_spec_list = []  # List of new tokens to construct new |spec|.
+  for item, start_id in coarse_id_assigner.GenStartIds():  # Topo-sorted..
+    new_id_list.append(str(start_id))
+    meta = item.meta
+    num_join = meta['join'].val if meta and 'join' in meta else 0
+    t = '*' * max(0, num_join - 1)
+    t += str(start_id)
+    t += '' if item.tags[0].usage == 10 else '+' + str(item.tags[0].usage)
+    new_spec_list.append((item.lo, t))
+    coarse_id_assigner.FeedWeight(item, item.tags[0].usage)
+  new_spec = ','.join(s for _, s in sorted(new_spec_list))
+  return ','.join(new_id_list), new_spec
+
+
+class AssignerUnittest(unittest.TestCase):
+
+  def testDagAssigner(self):
+    test_cases = [
+        # Trivial.
+        ('0', '0'),
+        ('137', '137'),
+        ('5,15', '5,6'),
+        ('11,18', '11+7,12'),
+        ('5,5', '5,5'),
+        # Series only.
+        ('0,10,20,30,40', '0,1,2,3,4'),
+        ('5,15,25,35,45,55', '5,6,7,8,9,10'),
+        ('5,15,25,35,45,55', '5,7,100,101,256,1001'),
+        ('0,10,20,45,85', '0,1,2+25,3+40,4'),
+        # Branching with and without join.
+        ('0,0,10,20,20,30,40', '0,0,1,2,2,3,4'),
+        ('0,0,10,20,20,30,40', '0,0,*1,2,2,*3,4'),
+        ('0,0,2,12,12,16,26', '0+4,0+2,1,2+8,2+4,3,4'),
+        ('0,0,4,14,14,22,32', '0+4,0+2,*1,2+8,2+4,*3,4'),
+        # Wide branching with and without join.
+        ('0,10,10,10,10,10,10,20,30', '0,1,1,1,1,1,1,2,3'),
+        ('0,10,10,10,10,10,10,20,30', '0,1,1,1,1,1,1,*****2,3'),
+        ('0,2,2,2,2,2,2,7,17', '0+2,1+4,1+19,1,1+4,1+2,1+5,2,3'),
+        ('0,2,2,2,2,2,2,21,31', '0+2,1+4,1+19,1,1+4,1+2,1+5,*****2,3'),
+        # Expanding different branch, without join.
+        ('0,10,10,10,60,70,80', '0,1+15,1+15,1+50,2,3,4'),
+        ('0,10,10,10,25,35,45', '0,1+15,1+50,1+15,2,3,4'),
+        ('0,10,10,10,25,35,45', '0,1+50,1+15,1+15,2,3,4'),
+        # ... with join.
+        ('0,10,10,10,60,70,80', '0,1+15,1+15,1+50,**2,3,4'),
+        ('0,10,10,10,60,70,80', '0,1+15,1+50,1+15,**2,3,4'),
+        ('0,10,10,10,60,70,80', '0,1+50,1+15,1+15,**2,3,4'),
+        # ... with alternative join.
+        ('0,10,10,10,60,70,80', '0,1+15,1+15,1+50,2,**3,4'),
+        ('0,10,10,10,25,60,70', '0,1+15,1+50,1+15,2,**3,4'),
+        ('0,10,10,10,25,60,70', '0,1+50,1+15,1+15,2,**3,4'),
+        # Examples from assigner.py.
+        ('0,10,10,20,0,10,20,30,0,10',
+         '0,1,1,*2,0,4,5,*6,0,7'),  # SA|AB|SDEC|SF
+        ('0,10,0,10,20,30', '0,1,0,2,*3,4'),  # SA|SB*CD
+        ('0,10,0,10,20,30', '0,1,0,2,3,*4'),  # SA|SBC*D
+        ('0,7,0,5,11,21', '0+7,1+4,0+5,2+3,*3,4'),  # SA|SB*CD
+        ('0,7,0,5,8,18', '0+7,1+4,0+5,2+3,3,*4'),  # SA|SBC*D
+        ('0,0,0,0,10,20', '0,0,0,0,*1,**2'),  # S|S|S|S*A**B
+        ('0,0,0,0,10,20', '0,0,0,0,**1,*2'),  # S|S|S|S**A*B
+        ('0,0,0,0,6,16', '0+8,0+7,0+6,0+5,*1,**2'),  # S|S|S|S*A**B
+        ('0,0,0,0,7,17', '0+8,0+7,0+6,0+5,**1,*2'),  # S|S|S|S**A*B
+        # Long branches without join.
+        ('0,10,0,0,10,20,0,10,20,30', '0,1,0,0,1,2,0,1,2,3'),
+        ('0,30,0,0,20,30,0,10,13,28', '0+30,1,0+50,0+20,1,2+17,0,1+3,2+15,3'),
+        # Long branches with join.
+        ('0,10,0,0,10,20,0,10,20,30', '0,1,0,0,1,2,0,1,2,***3'),
+        ('0,30,0,0,20,30,0,10,13,50',
+         '0+30,1,0+50,0+20,1,2+17,0,1+3,2+15,***3'),
+        # 2-level hierarchy.
+        ('0,10,10,20,0,10,10,20,30', '0,1,1,*2,0,1,1,*2,*3'),
+        ('0,2,2,10,0,3,3,6,34', '0+2,1+5,1+8,*2+24,0+3,1+2,1+3,*2+27,*3'),
+        ('0,2,2,10,0,3,3,6,34', '0+2,1+5,1+8,*2+24,0+3,1+2,1+3,*2+28,*3'),
+        ('0,2,2,10,0,3,3,6,35', '0+2,1+5,1+8,*2+24,0+3,1+2,1+3,*2+29,*3'),
+        # Binary hierarchy.
+        ('0,0,10,0,0,10,20,0,0,10,0,0,10,20,30',
+         '0,0,*1,0,0,*1,*2,0,0,*1,0,0,*1,*2,*3'),
+        ('0,0,2,0,0,5,11,0,0,8,0,0,5,14,18',
+         '0+1,0+2,*1+3,0+4,0+5,*1+6,*2+7,0+8,0+7,*1+6,0+5,0+4,*1+3,*2+2,*3+1'),
+        # Joining from different heads.
+        ('0,10,20,30,40,30,20,10,0,50', '0,1,2,3,4,3,2,1,0,****5'),
+        # Miscellaneous.
+        ('0,1,0,11', '0+1,1,0,*1'),
+    ]
+    for exp, spec in test_cases:
+      try:
+        actual, new_spec = _RunCoarseIdAssigner(spec)
+        self.failUnlessEqual(exp, actual)
+        # Test that assignment is idempotent.
+        actual2, new_spec2 = _RunCoarseIdAssigner(new_spec)
+        self.failUnlessEqual(actual, actual2)
+        self.failUnlessEqual(new_spec, new_spec2)
+      except Exception as e:
+        print(common.Color.RED(traceback.format_exc().rstrip()))
+        print('Failed spec: %s' % common.Color.CYAN(spec))
+        print('   Expected: %s' % common.Color.YELLOW(exp))
+        print('     Actual: %s' % common.Color.YELLOW(actual))
+        if new_spec != new_spec2:
+          print('Not idempotent')
+        if isinstance(e, AssertionError):
+          raise e
+
+
+if __name__ == '__main__':
+  unittest.main()

tools/grit/grit/tool/update_resource_ids/common.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+def AlignUp(v, align):
+  return (v + align - 1) // align * align
+
+
+def StripPlural(s):
+  assert s.endswith('s'), 'Expect %s to be plural' % s
+  return s[:-1]
+
+
+class Color:
+
+  def _MakeColor(code):
+    t = '\033[' + code + 'm%s\033[0m'
+    return lambda s: t % s
+
+  NONE = staticmethod(lambda s: s)
+  RED = staticmethod(_MakeColor('31'))
+  GREEN = staticmethod(_MakeColor('32'))
+  YELLOW = staticmethod(_MakeColor('33'))
+  BLUE = staticmethod(_MakeColor('34'))
+  MAGENTA = staticmethod(_MakeColor('35'))
+  CYAN = staticmethod(_MakeColor('36'))
+  WHITE = staticmethod(_MakeColor('37'))
+  GRAY = staticmethod(_MakeColor('30;1'))
+
+
+class TagInfo:
+  """Stores resource_ids tag entry (e.g., {"includes": 100} pair)."""
+
+  def __init__(self, raw_key, raw_value):
+    """TagInfo Constructor.
+
+    Args:
+      raw_key: parser.AnnotatedValue for the parsed key, e.g., "includes".
+      raw_value: parser.AnnotatedValue for the parsed value, e.g., 100.
+    """
+    # Tag name, e.g., 'include' (no "s" at end).
+    self.name = StripPlural(raw_key.val)
+    assert len(raw_value) == 1
+    # Inclusive start *position* of the tag's start ID in resource_ids.
+    self.lo = raw_value[0].lo
+    # Exclusive end *position* of the tag's start ID in resource_ids.
+    self.hi = raw_value[0].hi
+    # The tag's start ID. Initially the old value, but may be reassigned to new.
+    self.id = raw_value[0].val
+    # The number of IDs the tag uses, to be assigned by ItemInfo.SetUsages().
+    self.usage = None
+
+
+class ItemInfo:
+  """resource_ids item, containing multiple TagInfo."""
+
+  def __init__(self, lo, grd, raw_item):
+    # Inclusive start position of the item's key. Serve as unique identifier.
+    self.lo = lo
+    # The GRD filename for the item.
+    self.grd = grd
+    # Optional META information for the item.
+    self.meta = None
+    # List of TagInfo associated witih the item.
+    self.tags = []
+    for k, v in raw_item.items():
+      if k.val == 'META':
+        assert self.meta is None
+        self.meta = v  # Not flattened.
+      else:
+        self.tags.append(TagInfo(k, v))
+    self.tags.sort(key=lambda tag: tag.lo)
+
+  def SetUsages(self, tag_name_to_usage):
+    for tag in self.tags:
+      tag.usage = tag_name_to_usage.get(tag.name, 0)
+
+
+def BuildItemList(root_obj):
+  """Extracts ID assignments and structure from parsed resource_ids.
+
+  Returns: A list of ItemInfo, ordered by |lo|.
+  """
+  item_list = []
+  grd_seen = set()
+  for raw_key, raw_item in root_obj.iteritems():  # Unordered.
+    grd = raw_key.val
+    if grd == 'SRCDIR':
+      continue
+    if not grd.endswith('.grd'):
+      raise ValueError('Invalid GRD file: %s' % grd)
+    if grd in grd_seen:
+      raise ValueError('Duplicate GRD: %s' % grd)
+    grd_seen.add(grd)
+    item_list.append(ItemInfo(raw_key.lo, grd, raw_item))
+  item_list.sort(key=lambda item: item.lo)
+  return item_list

tools/grit/grit/tool/update_resource_ids/parser.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Structure-preserving parser for resource_ids files.
+
+Naive usage of eval() destroys resource_ids structure. This module provides a
+custom parser that annotates source byte ranges of "leaf values" (strings and
+integers).
+"""
+
+from __future__ import print_function
+
+_isWhitespace = lambda ch: ch in ' \t\n'
+_isNotNewline = lambda ch: ch != '\n'
+_isDigit = lambda ch: ch.isdigit()
+
+
+def _RenderLineCol(data, pos):
+  """Renders |pos| within text |data| in as text showing line and column."""
+  # This is used to pinpoint fatal parse errors, so okay to be inefficient.
+  new_lines = [i for i in range(pos) if data[i] == '\n']
+  row = 1 + len(new_lines)
+  col = (pos - new_lines[-1]) if new_lines else 1 + pos
+  return 'line %d, column %d' % (row, col)
+
+
+def Tokenize(data):
+  """Generator to split |data| into tokens.
+
+  Each token is specified as |(t, lo, hi)|:
+  * |t|: Type, with '#' = space / comments, '0' = int, 'S' = string, 'E' = end,
+         and other characters denoting themselves.
+  * |lo, hi|: Token's range within |data| (as |data[lo:hi]|).
+  """
+
+  class ctx:  # Local context for mutable data shared across inner functions.
+    pos = 0
+
+  def _HasData():
+    return ctx.pos < len(data)
+
+  # Returns True if ended by |not pred()|, or False if ended by EOF.
+  def _EatWhile(pred):
+    while _HasData():
+      if pred(data[ctx.pos]):
+        ctx.pos += 1
+      else:
+        return True
+    return False
+
+  def _NextBlank():
+    lo = ctx.pos
+    while True:
+      if not _EatWhile(_isWhitespace) or data[ctx.pos] != '#':
+        break
+      ctx.pos += 1
+      if not _EatWhile(_isNotNewline):
+        break
+      ctx.pos += 1
+    return None if ctx.pos == lo else (lo, ctx.pos)
+
+  def _EatString():
+    lo = ctx.pos
+    delim = data[ctx.pos]
+    is_escaped = False
+    ctx.pos += 1
+    while _HasData():
+      ch = data[ctx.pos]
+      ctx.pos += 1
+      if is_escaped:
+        is_escaped = False
+      elif ch == '\\':
+        is_escaped = True
+      elif ch == delim:
+        return
+    raise ValueError('Unterminated string at %s' % _RenderLineCol(data, lo))
+
+  while _HasData():
+    blank = _NextBlank()
+    if blank is not None:
+      yield ('#', blank[0], blank[1])
+    if not _HasData():
+      break
+    lo = ctx.pos
+    ch = data[ctx.pos]
+    if ch in '{}[],:':
+      ctx.pos += 1
+      t = ch
+    elif ch.isdigit():
+      _EatWhile(_isDigit)
+      t = '0'
+    elif ch in '+-':
+      ctx.pos += 1
+      if not _HasData() or not data[ctx.pos].isdigit():
+        raise ValueError('Invalid int at %s' % _RenderLineCol(data, lo))
+      _EatWhile(_isDigit)
+      t = '0'
+    elif ch in '"\'':
+      _EatString()
+      t = 'S'
+    else:
+      raise ValueError(
+          'Unknown char \'%s\' at %s' % (ch, _RenderLineCol(data, lo)))
+    yield (t, lo, ctx.pos)
+  yield ('E', ctx.pos, ctx.pos)  # End sentinel.
+
+
+def _SkipBlanks(toks):
+  """Generator to remove whitespace and comments from Tokenize()."""
+  for t, lo, hi in toks:
+    if t != '#':
+      yield t, lo, hi
+
+
+class AnnotatedValue:
+  """Container for leaf values (ints or strings) with an annotated range."""
+
+  def __init__(self, val, lo, hi):
+    self.val = val
+    self.lo = lo
+    self.hi = hi
+
+  def __str__(self):
+    return '<%s@%d:%d>' % (str(self.val), self.lo, self.hi)
+
+  def __repr__(self):
+    return '<%r@%d:%d>' % (self.val, self.lo, self.hi)
+
+  def __hash__(self):
+    return hash(self.val)
+
+  def __eq__(self, other):
+    return self.val == other
+
+
+class ResourceIdParser:
+  """resource_ids parser that stores leaf values as AnnotatedValue.
+
+  Algorithm: Use Tokenize() to split |data| into tokens and _SkipBlanks() to
+  ignore comments and spacing, then apply a recursive parsing, using a one-token
+  look-ahead for decision making.
+  """
+
+  def __init__(self, data, tok_gen):
+    self.data = data
+    self.state = []
+    self.toks = _SkipBlanks(tok_gen)
+    self.tok_look_ahead = None
+
+  def _MakeErr(self, msg, pos):
+    return ValueError(msg + ' at ' + _RenderLineCol(self.data, pos))
+
+  def _PeekTok(self):
+    if self.tok_look_ahead is None:
+      self.tok_look_ahead = self.toks.next()
+    return self.tok_look_ahead
+
+  def _NextTok(self):
+    if self.tok_look_ahead is None:
+      return self.toks.next()
+    ret = self.tok_look_ahead
+    self.tok_look_ahead = None
+    return ret
+
+  def _EatTok(self, exp_t, tok_name=None):
+    t, lo, _ = self._NextTok()
+    if t != exp_t:
+      raise self._MakeErr('Bad token: Expect \'%s\'' % (tok_name or exp_t), lo)
+
+  def _NextIntOrString(self):
+    t, lo, hi = self._NextTok()
+    if t != '0' and t != 'S':
+      raise self._MakeErr('Expected number or string', lo)
+    value = eval(self.data[lo:hi])
+    return AnnotatedValue(value, lo, hi)
+
+  # Consumes separator ',' and returns whether |end_ch| is encountered.
+  def _EatSep(self, end_ch):
+    t, lo, _ = self._PeekTok()
+    if t == ',':
+      self._EatTok(',')
+      # Allow trailing ','.
+      t, _, _ = self._PeekTok()
+      return t == end_ch
+    elif t == end_ch:
+      return True
+    else:
+      raise self._MakeErr('Expect \',\' or \'%s\'' % end_ch, lo)
+
+  def _NextList(self):
+    self._EatTok('[')
+    ret = []
+    t, _, _ = self._PeekTok()
+    if t != ']':
+      while True:
+        ret.append(self._NextObject())
+        if self._EatSep(']'):
+          break
+    self._EatTok(']')
+    return ret
+
+  def _NextDict(self):
+    self._EatTok('{')
+    ret = {}
+    t, _, _ = self._PeekTok()
+    if t != '}':
+      while True:
+        k = self._NextIntOrString()
+        self._EatTok(':')
+        v = self._NextObject()
+        ret[k] = v
+        if self._EatSep('}'):
+          break
+    self._EatTok('}')
+    return ret
+
+  def _NextObject(self):
+    t, lo, _ = self._PeekTok()
+    if t == '[':
+      return self._NextList()
+    elif t == '{':
+      return self._NextDict()
+    elif t == '0' or t == 'S':
+      return self._NextIntOrString()
+    else:
+      raise self._MakeErr('Bad token: Type = %s' % t, lo)
+
+  def Parse(self):
+    root_obj = self._NextObject()
+    self._EatTok('E', 'EOF')
+    return root_obj

tools/grit/grit/tool/update_resource_ids/reader.py

+# Copyright 2019 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Helpers to read GRD files and estimate resource ID usages.
+
+This module uses grit.grd_reader to estimate resource ID usages in GRD
+(and GRDP) files by counting the occurrences of {include, message, structure}
+tags. This approach avoids the complexties of conditional inclusions, but
+produces a conservative estimate of ID usages.
+"""
+
+from __future__ import print_function
+
+import collections
+import os
+
+from grit import grd_reader
+from grit.tool.update_resource_ids import common
+
+TAGS_OF_INTEREST = set(['include', 'message', 'structure'])
+
+
+def _CountResourceUsage(grd):
+  tag_name_to_count = {tag: set() for tag in TAGS_OF_INTEREST}
+  # Pass '_chromium', but '_google_chrome' would produce the same result.
+  root = grd_reader.Parse(grd, defines={'_chromium': True})
+  # Count all descendant tags, regardless of whether they're active.
+  for node in root.Preorder():
+    if node.name in TAGS_OF_INTEREST:
+      tag_name_to_count[node.name].add(node.attrs['name'])
+  return {k: len(v) for k, v in tag_name_to_count.iteritems() if v}
+
+
+def GenerateResourceUsages(item_list, src_dir, fake):
+  """Visits a list of ItemInfo to generate maps from tag name to usage.
+
+  Args:
+    root_obj: Root dict of a resource_ids file.
+    src_dir: Absolute directory of Chrome's src/ directory.
+    fake: For testing: Sets 10 as usages for all tags, to avoid reading GRD.
+  Yields:
+    Tuple (item, tag_name_to_usage), where |item| is from |item_list| and
+      |tag_name_to_usage| is a dict() mapping tag name to (int) usage.
+  """
+  if fake:
+    for item in item_list:
+      tag_name_to_usage = collections.Counter({t.name: 10 for t in item.tags})
+      yield item, tag_name_to_usage
+    return
+  for item in item_list:
+    supported_tag_names = set(tag.name for tag in item.tags)
+    if item.meta and 'sizes' in item.meta:
+      # If META has "sizes" field, use it instead of reading GRD.
+      tag_name_to_usage = collections.Counter()
+      for k, vlist in item.meta['sizes'].iteritems():
+        tag_name_to_usage[common.StripPlural(k.val)] = sum(v.val for v in vlist)
+      tag_names = set(tag_name_to_usage.keys())
+      if tag_names != supported_tag_names:
+        raise ValueError('META "sizes" field have identical fields as actual '
+                         '"sizes" field.')
+    else:
+      # Generated GRD start with '<(SHARED_INTERMEDIATE_DIR)'. Just check '<'.
+      if item.grd.startswith('<'):
+        raise ValueError('%s: Generated GRD must use META with "sizes" field '
+                         'to specify size bounds.' % item.grd)
+      grd_file = os.sep.join([src_dir, item.grd])
+      if not os.path.isfile(grd_file):
+        raise ValueError('Nonexistent GRD provided: %s' % item.grd)
+      tag_name_to_usage = _CountResourceUsage(grd_file)
+      tag_names = set(tag_name_to_usage.keys())
+      if not tag_names.issubset(supported_tag_names):
+        missing = [t + 's' for t in tag_names - supported_tag_names]
+        raise ValueError(
+            'Resource ids for %s needs entry for %s' % (item.grd, missing))
+    yield item, tag_name_to_usage

tools/gritsettings/resource_ids

@@ -19,6 +19,18 @@
 # - everything else
 #
 # The range of ID values, which is used by pak files, is from 0 to 2^16 - 1.
+#
+# IMPORTANT: Update instructions:
+# * If adding items, manually assign draft start IDs so that numerical order is
+#   preserved. Usually it suffices to +1 from previous tag.
+#   * If updating items with repeated, be sure to add / update
+#     "META": {"join": <duplicate count>},
+#     for the item following duplicates. Be sure to look for duplicates that
+#     may appear earlier than those that immediately precede the item.
+# * To update IDs, run the following (from src/):
+#
+#   python tools/grit/grit.py update_resource_ids \
+#        -o tools/gritsettings/resource_ids tools/gritsettings/resource_ids
 {
   # The first entry in the file, SRCDIR, is special: It is a relative path from
   # this file to the base of your checkout.
@@ -43,6 +55,7 @@
   # Leave lots of space for generated_resources since it has most of our
   # strings.
   "chrome/app/generated_resources.grd": {
+    "META": {"join": 2},
     "messages": [800],
   },
 
@@ -69,6 +82,7 @@
   },
 
   "chrome/app/theme/chrome_unscaled_resources.grd": {
+    "META": {"join": 5},
     "includes": [10150],
   },
 
@@ -215,16 +229,19 @@
   "chromeos/components/media_app_ui/resources/media_app_resources.grd": {
     "includes": [14680],
   },
-  # Both media_app_bundle_resources.grd and media_app_bundle_mock_resources.grd start
-  # with the same id because only one of them is built depending on if src_internal is
-  # available. Lower bound for number of resource ids is number of languages (74).
+  # Both media_app_bundle_resources.grd and media_app_bundle_mock_resources.grd
+  # start with the same id because only one of them is built depending on if
+  # src_internal is available. Lower bound for number of resource ids is number
+  # of languages (74).
   "chromeos/components/media_app_ui/resources/app/app/media_app_bundle_resources.grd": {
+    "META": {"sizes": {"includes": [100],}},  # Relies on src-internal.
     "includes": [14690],
   },
   "chromeos/components/media_app_ui/resources/mock/media_app_bundle_mock_resources.grd": {
     "includes": [14690],
   },
   "chromeos/resources/chromeos_resources.grd": {
+    "META": {"join": 2},
     "includes": [14790],
   },
   # END chromeos/ section.
@@ -287,6 +304,7 @@
   },
 
   "ios/chrome/app/strings/ios_strings.grd": {
+    "META": {"join": 2},
     "messages": [2000],
   },
   "ios/chrome/app/theme/ios_theme_resources.grd": {
@@ -305,6 +323,7 @@
     "messages": [3070],
   },
   "ios/chrome/content_widget_extension/strings/ios_content_widget_extension_chromium_strings.grd": {
+    "META": {"join": 2},
     "messages": [3080],
   },
   "ios/chrome/content_widget_extension/strings/ios_content_widget_extension_google_chrome_strings.grd": {
@@ -316,6 +335,7 @@
   # content/ and ios/web/ must start at the same id.
   # App only use one file depending on whether it is iOS or other platform.
   "content/app/resources/content_resources.grd": {
+    "META": {"join": 3},
     "structures": [20000],
   },
   "content/browser/webrtc/resources/resources.grd": {
@@ -330,6 +350,7 @@
 
   # This file is generated during the build.
   "<(SHARED_INTERMEDIATE_DIR)/content/browser/tracing/tracing_resources.grd": {
+    "META": {"sizes": {"includes": [20],}},
     "includes": [20550],
   },
   # END content/ section.
@@ -483,6 +504,7 @@
 
   # This file is generated during the build.
   "<(SHARED_INTERMEDIATE_DIR)/devtools/devtools_resources.grd": {
+    "META": {"sizes": {"includes": [500],}},
     "includes": [28880],
   },