[tools/perf] Read measurements from artifacts

This CL implements reading ad-hoc measurements stored in a special
artifact written by Telemetry and converting them to histograms.

Bug: 981349
Change-Id: Ie6d88665394503a306e1c6d6ceefcc9b75319ecb
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1849383Reviewed-by: Juan Antonio Navarro Pérez <perezju@chromium.org>
Commit-Queue: Mikhail Khokhlov <khokhlov@google.com>
Cr-Commit-Position: refs/heads/master@{#705901}

tools/perf/core/results_processor/compute_metrics.py

@@ -60,8 +60,9 @@ def ComputeTBMv2Metrics(intermediate_results):
   """Compute metrics on aggregated traces in parallel.
 
   For each test run that has an aggregate trace and some TBMv2 metrics listed
-  in its tags, compute the metrics and store the result as histogram dicts
-  in the corresponding test result.
+  in its tags, compute the metrics and return the list of all resulting
+  histograms. Note: the order of histograms in the results may be different
+  from the order of tests in intermediate_results.
   """
   histogram_dicts = []
   work_list = []

tools/perf/core/results_processor/formatters/json3_output.py

@@ -8,13 +8,13 @@ Format specification:
 https://chromium.googlesource.com/chromium/src/+/master/docs/testing/json_test_results_format.md
 """
 
-import calendar
 import collections
-import datetime
 import json
 import os
 import urllib
 
+from core.results_processor import util
+
 
 OUTPUT_FILENAME = 'test-results.json'
 
@@ -79,7 +79,7 @@ def Convert(in_results, base_dir):
 
   benchmark_run = in_results['benchmarkRun']
   results.update(
-      seconds_since_epoch=_TimestampToEpoch(benchmark_run['startTime']),
+      seconds_since_epoch=util.IsoTimestampToEpoch(benchmark_run['startTime']),
       interrupted=benchmark_run['interrupted'],
       num_failures_by_type=dict(status_counter),
       path_delimiter='/',
@@ -122,12 +122,6 @@ def _ArtifactPath(artifact, base_dir):
     return path.replace(os.sep, '/')
 
 
-def _TimestampToEpoch(timestamp):
-  """Convert UTC timestamp to seconds since epoch with microsecond precision."""
-  dt = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
-  return calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6
-
-
 def _MergeDict(target, values):
   # Is used to merge multiple runs of a story into a single test result.
   for key, value in values.items():

tools/perf/core/results_processor/processor.py

@@ -21,11 +21,15 @@ from core.results_processor import formatters
 from core.results_processor import util
 
 from tracing.trace_data import trace_data
+from tracing.value.diagnostics import date_range
 from tracing.value.diagnostics import generic_set
 from tracing.value.diagnostics import reserved_infos
+from tracing.value import histogram
 from tracing.value import histogram_set
+from tracing.value import legacy_unit_info
 
 TELEMETRY_RESULTS = '_telemetry_results.jsonl'
+MEASUREMENTS_NAME = 'measurements.json'
 
 FORMATS_WITH_METRICS = ['csv', 'histograms', 'html']
 
@@ -179,6 +183,7 @@ def UploadArtifacts(intermediate_results, upload_bucket, results_label):
 
 def _ComputeMetrics(intermediate_results, results_label):
   histogram_dicts = compute_metrics.ComputeTBMv2Metrics(intermediate_results)
+  histogram_dicts += ExtractMeasurements(intermediate_results)
   histogram_dicts = AddDiagnosticsToHistograms(
       histogram_dicts, intermediate_results, results_label)
   return histogram_dicts
@@ -202,6 +207,68 @@ def AddDiagnosticsToHistograms(histogram_dicts, intermediate_results,
         reserved_infos.LABELS.name,
         generic_set.GenericSet([results_label]))
 
+  histograms.DeduplicateDiagnostics()
+  return histograms.AsDicts()
+
+
+def MeasurementToHistogram(name, measurement):
+  unit = measurement['unit']
+  samples = measurement['samples']
+  description = measurement.get('description')
+  if unit in legacy_unit_info.LEGACY_UNIT_INFO:
+    info = legacy_unit_info.LEGACY_UNIT_INFO[unit]
+    unit = info.name
+    samples = [s * info.conversion_factor for s in samples]
+  if unit not in histogram.UNIT_NAMES:
+    raise ValueError('Unknown unit: %s' % unit)
+  return histogram.Histogram.Create(name, unit, samples,
+                                    description=description)
+
+
+def _GlobalDiagnostics(benchmark_run):
+  """Extract diagnostics information about the whole benchmark run.
+
+  These diagnostics will be added to ad-hoc measurements recorded by
+  benchmarks.
+  """
+  timestamp_ms = util.IsoTimestampToEpoch(benchmark_run['startTime']) * 1e3
+  return {
+    reserved_infos.BENCHMARK_START.name: date_range.DateRange(timestamp_ms),
+  }
+
+
+def _StoryDiagnostics(test_result):
+  """Extract diagnostics information about the specific story.
+
+  These diagnostics will be added to ad-hoc measurements recorded by
+  benchmarks.
+  """
+  benchmark_name, story_name = test_result['testPath'].split('/', 1)
+  story_tags = [tag['value'] for tag in test_result.get('tags', [])
+                if tag['key'] == 'story_tag']
+  return {
+      reserved_infos.BENCHMARKS.name: generic_set.GenericSet([benchmark_name]),
+      reserved_infos.STORIES.name: generic_set.GenericSet([story_name]),
+      reserved_infos.STORY_TAGS.name: generic_set.GenericSet(story_tags),
+  }
+
+
+def ExtractMeasurements(intermediate_results):
+  """Add ad-hoc measurements to histogram dicts"""
+  histograms = histogram_set.HistogramSet()
+  global_diagnostics = _GlobalDiagnostics(intermediate_results['benchmarkRun'])
+
+  for result in intermediate_results['testResults']:
+    artifacts = result.get('outputArtifacts', {})
+    if MEASUREMENTS_NAME in artifacts:
+      with open(artifacts[MEASUREMENTS_NAME]['filePath']) as f:
+        measurements = json.load(f)['measurements']
+      diagnostics = global_diagnostics.copy()
+      diagnostics.update(_StoryDiagnostics(result))
+      for name, measurement in measurements.iteritems():
+        histograms.AddHistogram(MeasurementToHistogram(name, measurement),
+                                diagnostics=diagnostics)
+
   return histograms.AsDicts()
 
 

tools/perf/core/results_processor/processor_test.py

@@ -10,6 +10,7 @@ with their expected contents.
 """
 
 import csv
+import datetime
 import json
 import os
 import shutil
@@ -25,6 +26,7 @@ from core.results_processor import processor
 from core.results_processor import testing
 
 from tracing.value.diagnostics import generic_set
+from tracing.value.diagnostics import date_range
 from tracing.value import histogram
 from tracing.value import histogram_set
 from tracing_build import render_histograms_viewer
@@ -309,6 +311,73 @@ class ResultsProcessorIntegrationTests(unittest.TestCase):
     self.assertIsNotNone(hist)
     self.assertIn('traceUrls', hist.diagnostics)
 
+  def testHistogramsOutputMeasurements(self):
+    measure_file = os.path.join(self.output_dir,
+                                processor.MEASUREMENTS_NAME)
+    with open(measure_file, 'w') as f:
+      json.dump({'measurements': {
+          'a': {'unit': 'ms', 'samples': [4, 6], 'description': 'desc_a'},
+          'b': {'unit': 'ms', 'samples': [5], 'description': 'desc_b'},
+      }}, f)
+
+    start_ts = 1500000000
+    start_iso = datetime.datetime.utcfromtimestamp(start_ts).isoformat() + 'Z'
+
+    self.SerializeIntermediateResults(
+        test_results=[
+            testing.TestResult(
+                'benchmark/story',
+                output_artifacts={
+                    processor.MEASUREMENTS_NAME: testing.Artifact(measure_file)
+                },
+                tags=['story_tag:test']
+            ),
+        ],
+        start_time=start_iso,
+    )
+
+    processor.main([
+        '--output-format', 'histograms',
+        '--output-dir', self.output_dir,
+        '--intermediate-dir', self.intermediate_dir,
+    ])
+
+    with open(os.path.join(
+        self.output_dir, histograms_output.OUTPUT_FILENAME)) as f:
+      results = json.load(f)
+
+    out_histograms = histogram_set.HistogramSet()
+    out_histograms.ImportDicts(results)
+    self.assertEqual(len(out_histograms), 2)
+
+    hist = out_histograms.GetHistogramNamed('a')
+    self.assertEqual(hist.name, 'a')
+    self.assertEqual(hist.unit, 'ms_smallerIsBetter')
+    self.assertEqual(hist.sample_values, [4, 6])
+    self.assertEqual(hist.description, 'desc_a')
+    self.assertEqual(hist.diagnostics['benchmarks'],
+                     generic_set.GenericSet(['benchmark']))
+    self.assertEqual(hist.diagnostics['stories'],
+                     generic_set.GenericSet(['story']))
+    self.assertEqual(hist.diagnostics['storyTags'],
+                     generic_set.GenericSet(['test']))
+    self.assertEqual(hist.diagnostics['benchmarkStart'],
+                     date_range.DateRange(start_ts * 1e3))
+
+    hist = out_histograms.GetHistogramNamed('b')
+    self.assertEqual(hist.name, 'b')
+    self.assertEqual(hist.unit, 'ms_smallerIsBetter')
+    self.assertEqual(hist.sample_values, [5])
+    self.assertEqual(hist.description, 'desc_b')
+    self.assertEqual(hist.diagnostics['benchmarks'],
+                     generic_set.GenericSet(['benchmark']))
+    self.assertEqual(hist.diagnostics['stories'],
+                     generic_set.GenericSet(['story']))
+    self.assertEqual(hist.diagnostics['storyTags'],
+                     generic_set.GenericSet(['test']))
+    self.assertEqual(hist.diagnostics['benchmarkStart'],
+                     date_range.DateRange(start_ts * 1e3))
+
   def testHtmlOutput(self):
     hist_file = os.path.join(self.output_dir,
                              compute_metrics.HISTOGRAM_DICTS_FILE)

tools/perf/core/results_processor/processor_unittest.py

@@ -147,3 +147,29 @@ class ResultsProcessorUnitTests(unittest.TestCase):
       artifacts = result['outputArtifacts']
       self.assertEqual(len(artifacts), 1)
       self.assertEqual(artifacts.keys()[0], 'trace.html')
+
+  def testMeasurementToHistogram(self):
+    hist = processor.MeasurementToHistogram('a', {
+      'unit': 'sizeInBytes',
+      'samples': [1, 2, 3],
+      'description': 'desc',
+    })
+
+    self.assertEqual(hist.name, 'a')
+    self.assertEqual(hist.unit, 'sizeInBytes')
+    self.assertEqual(hist.sample_values, [1, 2, 3])
+    self.assertEqual(hist.description, 'desc')
+
+  def testMeasurementToHistogramLegacyUnits(self):
+    hist = processor.MeasurementToHistogram('a', {
+      'unit': 'seconds',
+      'samples': [1, 2, 3],
+    })
+
+    self.assertEqual(hist.name, 'a')
+    self.assertEqual(hist.unit, 'ms_smallerIsBetter')
+    self.assertEqual(hist.sample_values, [1000, 2000, 3000])
+
+  def testMeasurementToHistogramUnknownUnits(self):
+    with self.assertRaises(ValueError):
+      processor.MeasurementToHistogram('a', {'unit': 'yards', 'samples': [9]})

tools/perf/core/results_processor/util.py

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import calendar
+import datetime
 import logging
 import multiprocessing
 from multiprocessing.dummy import Pool as ThreadPool
@@ -48,3 +50,12 @@ def ApplyInParallel(function, work_list):
     pool.join()
   finally:
     pool.terminate()
+
+
+def IsoTimestampToEpoch(timestamp):
+  """Convert ISO formatted time to seconds since epoch."""
+  try:
+    dt = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%S.%fZ')
+  except ValueError:
+    dt = datetime.datetime.strptime(timestamp, '%Y-%m-%dT%H:%M:%SZ')
+  return calendar.timegm(dt.timetuple()) + dt.microsecond / 1e6