Add histograms_index file to record all available histograms xmls

This histograms_index file provides a convenient way for a job to know
all of the histogram files without having to search for them. As we
split up histograms, we need a way to locate all available histograms
xmls and thus we need this index file.

This cl also adds a PRESUBMIT check to make sure this index file is
up-to-date.

Bug: 1116807
Change-Id: Ifc55e421092448b5cecaa573f02e651a098428bf
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2359234Reviewed-by: Steven Holte <holte@chromium.org>
Reviewed-by: Brian White <bcwhite@chromium.org>
Commit-Queue: Weilun Shi <sweilun@chromium.org>
Auto-Submit: Weilun Shi <sweilun@chromium.org>
Cr-Commit-Position: refs/heads/master@{#800357}

tools/metrics/common/path_util.py

@@ -7,15 +7,6 @@
 import os.path
 
 
-def GetHistogramsFile():
-  """Returns the path to histograms.xml.
-
-  Prefer using this function instead of just open("histograms.xml"), so that
-  scripts work properly even if run from outside the histograms directory.
-  """
-  return GetInputFile('tools/metrics/histograms/histograms.xml')
-
-
 def GetInputFile(src_relative_file_path):
   """Converts a src/-relative file path into a path that can be opened."""
   depth = [os.path.dirname(__file__), '..', '..', '..']

tools/metrics/histograms/histogram_paths.py

-# Copyright 2017 The Chromium Authors. All rights reserved.
+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
@@ -11,28 +12,29 @@ sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))
 import path_util
 
 
-def _FindHistogramsXmlFiles(base_dir):
-  """Gets a list of all histograms.xml files under the directory tree, prefixed
-  with tools/metrics/histograms/."""
+def _FindHistogramsXmlFiles():
+  """Gets a list relative path to all histograms xmls under histograms_xml/."""
   file_list = []
-  for dirName, _, fileList in os.walk(base_dir):
+  for dirName, _, fileList in os.walk(PATH_TO_HISTOGRAMS_XML_DIR):
     for filename in fileList:
-      if filename == 'histograms.xml':
-        filepath = os.path.join('tools/metrics/histograms/', dirName, filename)
-        file_list.append(filepath)
+      if filename == 'histograms.xml' or filename == 'histogram_suffixes.xml':
+        # Compute the relative path of the histograms xml file.
+        filepath = os.path.relpath(
+          os.path.join(dirName, filename), PATH_TO_HISTOGRAMS_XML_DIR)
+        file_list.append(os.path.join('tools/metrics/histograms/', filepath))
   return file_list
 
 
 ENUMS_XML_RELATIVE = 'tools/metrics/histograms/enums.xml'
-# This file path accounts for cases where the script is executed from other
-# metrics-related directories.
-PATH_TO_HISTOGRAMS_XML_DIR = os.path.join('..', 'histograms/histograms_xml')
+# The absolute path to the histograms_xml folder.
+PATH_TO_HISTOGRAMS_XML_DIR = path_util.GetInputFile(
+    'tools/metrics/histograms/histograms_xml')
 # In the middle state, histogram paths include both the large histograms.xml
 # file as well as the split up files.
 # TODO: Improve on the current design to avoid calling `os.walk()` at the time
 # of module import.
 HISTOGRAMS_XMLS_RELATIVE = (['tools/metrics/histograms/histograms.xml'] +
-                            _FindHistogramsXmlFiles(PATH_TO_HISTOGRAMS_XML_DIR))
+                            _FindHistogramsXmlFiles())
 OBSOLETE_XML_RELATIVE = ('tools/metrics/histograms/histograms_xml/'
                          'obsolete_histograms.xml')
 ALL_XMLS_RELATIVE = [ENUMS_XML_RELATIVE, OBSOLETE_XML_RELATIVE
@@ -42,7 +44,7 @@ ENUMS_XML = path_util.GetInputFile(ENUMS_XML_RELATIVE)
 UKM_XML = path_util.GetInputFile('tools/metrics/ukm/ukm.xml')
 HISTOGRAMS_XMLS = [path_util.GetInputFile(f) for f in HISTOGRAMS_XMLS_RELATIVE]
 OBSOLETE_XML = path_util.GetInputFile(OBSOLETE_XML_RELATIVE)
-ALL_XMLS = [ENUMS_XML, OBSOLETE_XML] + HISTOGRAMS_XMLS
+ALL_XMLS = [path_util.GetInputFile(f) for f in ALL_XMLS_RELATIVE]
 
 ALL_TEST_XMLS_RELATIVE = [
     'tools/metrics/histograms/test_data/enums.xml',
@@ -51,3 +53,18 @@ ALL_TEST_XMLS_RELATIVE = [
 ]
 ALL_TEST_XMLS = [path_util.GetInputFile(f) for f in ALL_TEST_XMLS_RELATIVE]
 TEST_ENUMS_XML, TEST_HISTOGRAMS_XML, TEST_UKM_XML = ALL_TEST_XMLS
+
+# The path to the `histogram_index` file.
+HISTOGRAMS_INDEX = path_util.GetInputFile(
+    'tools/metrics/histograms/histograms_index.txt')
+
+
+def main():
+  with open(HISTOGRAMS_INDEX, 'w+') as f:
+    f.write("\n".join(HISTOGRAMS_XMLS_RELATIVE))
+
+
+if __name__ == '__main__':
+  # Update the `histograms_index` file whenever histograms paths are updated.
+  # This file records all currently existing histograms.xml paths.
+  main()

tools/metrics/histograms/histograms_index.txt

+tools/metrics/histograms/histograms.xml
+tools/metrics/histograms/histogram_suffixes.xml
+tools/metrics/histograms/Fingerprint/histograms.xml
+tools/metrics/histograms/UMA/histograms.xml
\ No newline at end of file

tools/metrics/histograms/validate_histograms_index.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Verify the `histograms_index` file is up-to-date."""
+
+import logging
+import os
+import sys
+
+import histogram_paths
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'common'))
+import path_util
+
+
+def main()
+  exit_code = 0
+  with open(histogram_paths.HISTOGRAMS_INDEX, 'r') as f:
+    histograms_paths = [line.strip() for line in f]
+    if histograms_paths != histogram_paths.HISTOGRAMS_XMLS_RELATIVE:
+      exit_code = 1
+      logging.error(
+        'histograms_index.txt is not up-to-date. Please run '
+        'python histogram_paths.py to update it.')
+
+  sys.exit(exit_code)
+
+
+if __name__ == '__main__':
+  main()
\ No newline at end of file