Add Chart JSON processing to Telemetry.

This patch implements Chart JSON for Telemetry, which represents a JSON format specifically for conveying data to Buildbot to be sent to the perf dashboard in charted/summarized form. 
Consult go/telemetry-json for further specs on this format.
This is an 0.1 since the overall metadata and description portions of the format are still in flux.

BUG=399970

Review URL: https://codereview.chromium.org/439613003

Cr-Commit-Position: refs/heads/master@{#291185}
git-svn-id: svn://svn.chromium.org/chrome/trunk/src@291185 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/results/buildbot_output_formatter.py

@@ -52,7 +52,7 @@ class BuildbotOutputFormatter(output_formatter.OutputFormatter):
       return
 
     buildbot_measurement_name, buildbot_trace_name = (
-        value.GetBuildbotMeasurementAndTraceNameForPerPageResult())
+        value.GetChartAndTraceNameForPerPageResult())
     self._PrintPerfResult(buildbot_measurement_name,
                           buildbot_trace_name,
                           buildbot_value, value.units, buildbot_data_type)
@@ -71,7 +71,7 @@ class BuildbotOutputFormatter(output_formatter.OutputFormatter):
       return
 
     buildbot_measurement_name, buildbot_trace_name = (
-        value.GetBuildbotMeasurementAndTraceNameForComputedSummaryResult(
+        value.GetChartAndTraceNameForComputedSummaryResult(
             self._trace_tag))
     self._PrintPerfResult(buildbot_measurement_name,
                           buildbot_trace_name,
@@ -87,7 +87,7 @@ class BuildbotOutputFormatter(output_formatter.OutputFormatter):
         buildbot_data_type = value.GetBuildbotDataType(
             output_context=value_module.SUMMARY_RESULT_OUTPUT_CONTEXT)
         buildbot_measurement_name, buildbot_trace_name = (
-            value.GetBuildbotMeasurementAndTraceNameForComputedSummaryResult(
+            value.GetChartAndTraceNameForComputedSummaryResult(
                 self._trace_tag))
         self._PrintPerfResult(
             buildbot_measurement_name,

tools/telemetry/telemetry/results/chart_json.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import itertools
+
+from telemetry.value import summary as summary_module
+
+def ResultsAsChartDict(benchmark_metadata, page_specific_values,
+                       summary_values):
+  """Produces a dict for serialization to Chart JSON format from raw values.
+
+  Chart JSON is a transformation of the basic Telemetry JSON format that
+  removes the page map, summarizes the raw values, and organizes the results
+  by chart and trace name. This function takes the key pieces of data needed to
+  perform this transformation (namely, lists of values and a benchmark metadata
+  object) and processes them into a dict which can be serialized using the json
+  module.
+
+  Design doc for schema: http://goo.gl/kOtf1Y
+
+  Args:
+    page_specific_values: list of page-specific values
+    summary_values: list of summary values
+    benchmark_metadata: a benchmark.BenchmarkMetadata object
+
+  Returns:
+    A Chart JSON dict corresponding to the given data.
+  """
+  summary = summary_module.Summary(page_specific_values)
+  values = itertools.chain(
+      summary.interleaved_computed_per_page_values_and_summaries,
+      summary_values)
+  charts = collections.defaultdict(dict)
+
+  for value in values:
+    if value.page:
+      chart_name, trace_name = (
+          value.GetChartAndTraceNameForPerPageResult())
+    else:
+      chart_name, trace_name = (
+          value.GetChartAndTraceNameForComputedSummaryResult(None))
+      if chart_name == trace_name:
+        trace_name = 'summary'
+
+    assert trace_name not in charts[chart_name]
+
+    charts[chart_name][trace_name] = value.AsDict()
+
+  result_dict = {
+    'format_version': '0.1',
+    'benchmark_name': benchmark_metadata.name,
+    'charts': charts
+  }
+
+  return result_dict

tools/telemetry/telemetry/results/chart_json_unittest.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import unittest
+import json
+
+from telemetry import benchmark
+from telemetry.results import chart_json
+from telemetry.page import page_set
+from telemetry.value import scalar
+from telemetry.value import list_of_scalar_values
+
+
+def _MakePageSet():
+  ps = page_set.PageSet(file_path=os.path.dirname(__file__))
+  ps.AddPageWithDefaultRunNavigate('http://www.foo.com/')
+  ps.AddPageWithDefaultRunNavigate('http://www.bar.com/')
+  return ps
+
+class ChartJsonTest(unittest.TestCase):
+  def setUp(self):
+    self._page_set = _MakePageSet()
+    self._benchmark_metadata = benchmark.BenchmarkMetadata('benchmark_name')
+
+  def testAsChartDictSerializable(self):
+    v0 = scalar.ScalarValue(self._page_set[0], 'foo', 'seconds', 3)
+    page_specific_values = [v0]
+    summary_values = []
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+    json.dumps(d)
+
+  def testAsChartDictBaseKeys(self):
+    page_specific_values = []
+    summary_values = []
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+
+    self.assertEquals(d['format_version'], '0.1')
+    self.assertEquals(d['benchmark_name'], 'benchmark_name')
+
+  def testAsChartDictPageSpecificValuesSamePage(self):
+    v0 = scalar.ScalarValue(self._page_set[0], 'foo', 'seconds', 3)
+    v1 = scalar.ScalarValue(self._page_set[0], 'foo', 'seconds', 4)
+    page_specific_values = [v0, v1]
+    summary_values = []
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+
+    self.assertTrue('foo' in d['charts'])
+    self.assertTrue('http://www.foo.com/' in d['charts']['foo'])
+
+  def testAsChartDictPageSpecificValuesAndComputedSummary(self):
+    v0 = scalar.ScalarValue(self._page_set[0], 'foo', 'seconds', 3)
+    v1 = scalar.ScalarValue(self._page_set[1], 'foo', 'seconds', 4)
+    page_specific_values = [v0, v1]
+    summary_values = []
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+
+    self.assertTrue('foo' in d['charts'])
+    self.assertTrue('http://www.foo.com/' in d['charts']['foo'])
+    self.assertTrue('http://www.bar.com/' in d['charts']['foo'])
+    self.assertTrue('summary' in d['charts']['foo'])
+
+  def testAsChartDictSummaryValueWithTraceName(self):
+    v0 = list_of_scalar_values.ListOfScalarValues(None, 'foo.bar', 'seconds',
+        [3, 4])
+    page_specific_values = []
+    summary_values = [v0]
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+
+    self.assertTrue('bar' in d['charts']['foo'])
+
+  def testAsChartDictSummaryValueWithoutTraceName(self):
+    v0 = list_of_scalar_values.ListOfScalarValues(None, 'foo', 'seconds',
+        [3, 4])
+    page_specific_values = []
+    summary_values = [v0]
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+
+    self.assertTrue('summary' in d['charts']['foo'])
+
+  def testAsChartDictValueSmokeTest(self):
+    v0 = list_of_scalar_values.ListOfScalarValues(None, 'foo.bar', 'seconds',
+        [3, 4])
+    page_specific_values = []
+    summary_values = [v0]
+
+    d = chart_json.ResultsAsChartDict(
+        self._benchmark_metadata,
+        page_specific_values,
+        summary_values)
+
+    self.assertEquals(d['charts']['foo']['bar']['values'], [3, 4])

tools/telemetry/telemetry/results/json_output_formatter.py

@@ -6,16 +6,26 @@ import json
 
 from telemetry.results import output_formatter
 
-
-def ResultsAsDict(page_test_results, metadata):
+def ResultsAsDict(page_test_results, benchmark_metadata):
+  """Takes PageTestResults to a dict serializable to JSON.
+
+  To serialize results as JSON we first convert them to a dict that can be
+  serialized by the json module. It also requires a benchmark_metadat object
+  for metadata to be integrated into the results (currently the benchmark
+  name).
+
+  Args:
+    page_test_results: a PageTestResults object
+    benchmark_metadata: a benchmark.BenchmarkMetadata object
+  """
   result_dict = {
     'format_version': '0.2',
-    'benchmark_name': metadata.name,
+    'benchmark_name': benchmark_metadata.name,
     'summary_values': [v.AsDict() for v in
                        page_test_results.all_summary_values],
     'per_page_values': [v.AsDict() for v in
                         page_test_results.all_page_specific_values],
-    'pages': dict((p.id, p.AsDict()) for p in _GetAllPages(page_test_results))
+    'pages': {p.id: p.AsDict() for p in _GetAllPages(page_test_results)}
   }
 
   return result_dict
@@ -26,15 +36,15 @@ def _GetAllPages(page_test_results):
   return pages
 
 class JsonOutputFormatter(output_formatter.OutputFormatter):
-  def __init__(self, output_stream, metadata):
+  def __init__(self, output_stream, benchmark_metadata):
     super(JsonOutputFormatter, self).__init__(output_stream)
-    self._metadata = metadata
+    self._benchmark_metadata = benchmark_metadata
 
   @property
-  def metadata(self):
-    return self._metadata
+  def benchmark_metadata(self):
+    return self._benchmark_metadata
 
   def Format(self, page_test_results):
-    json.dump(ResultsAsDict(page_test_results, self.metadata),
+    json.dump(ResultsAsDict(page_test_results, self.benchmark_metadata),
         self.output_stream)
     self.output_stream.write('\n')

tools/telemetry/telemetry/results/json_output_formatter_unittest.py

@@ -30,7 +30,7 @@ class JsonOutputFormatterTest(unittest.TestCase):
     self._output = StringIO.StringIO()
     self._page_set = _MakePageSet()
     self._formatter = json_output_formatter.JsonOutputFormatter(self._output,
-        benchmark.BenchmarkMetadata('test_name'))
+        benchmark.BenchmarkMetadata('benchmark_name'))
 
   def testOutputAndParse(self):
     results = page_test_results.PageTestResults()
@@ -45,6 +45,14 @@ class JsonOutputFormatterTest(unittest.TestCase):
     self._formatter.Format(results)
     json.loads(self._output.getvalue())
 
+  def testAsDictBaseKeys(self):
+    results = page_test_results.PageTestResults()
+    d = json_output_formatter.ResultsAsDict(results,
+        self._formatter.benchmark_metadata)
+
+    self.assertEquals(d['format_version'], '0.2')
+    self.assertEquals(d['benchmark_name'], 'benchmark_name')
+
   def testAsDictWithOnePage(self):
     results = page_test_results.PageTestResults()
     results.WillRunPage(self._page_set[0])
@@ -52,7 +60,8 @@ class JsonOutputFormatterTest(unittest.TestCase):
     results.AddValue(v0)
     results.DidRunPage(self._page_set[0])
 
-    d = json_output_formatter.ResultsAsDict(results, self._formatter.metadata)
+    d = json_output_formatter.ResultsAsDict(results,
+        self._formatter.benchmark_metadata)
 
     self.assertTrue(_HasPage(d['pages'], self._page_set[0]))
     self.assertTrue(_HasValueNamed(d['per_page_values'], 'foo'))
@@ -69,7 +78,8 @@ class JsonOutputFormatterTest(unittest.TestCase):
     results.AddValue(v1)
     results.DidRunPage(self._page_set[1])
 
-    d = json_output_formatter.ResultsAsDict(results, self._formatter.metadata)
+    d = json_output_formatter.ResultsAsDict(results,
+        self._formatter.benchmark_metadata)
 
     self.assertTrue(_HasPage(d['pages'], self._page_set[0]))
     self.assertTrue(_HasPage(d['pages'], self._page_set[1]))
@@ -81,7 +91,8 @@ class JsonOutputFormatterTest(unittest.TestCase):
     v = scalar.ScalarValue(None, 'baz', 'seconds', 5)
     results.AddSummaryValue(v)
 
-    d = json_output_formatter.ResultsAsDict(results, self._formatter.metadata)
+    d = json_output_formatter.ResultsAsDict(results,
+        self._formatter.benchmark_metadata)
 
     self.assertFalse(d['pages'])
     self.assertTrue(_HasValueNamed(d['summary_values'], 'baz'))

tools/telemetry/telemetry/results/results_options.py

@@ -16,8 +16,8 @@ from telemetry.results import page_test_results
 from telemetry.results import progress_reporter
 
 # Allowed output formats. The default is the first item in the list.
-_OUTPUT_FORMAT_CHOICES = ('html', 'buildbot', 'block', 'csv', 'gtest',
-    'json', 'none')
+_OUTPUT_FORMAT_CHOICES = ('html', 'buildbot', 'block', 'csv', 'gtest', 'json',
+    'none')
 
 
 def AddResultsOptions(parser):

tools/telemetry/telemetry/value/__init__.py

@@ -130,9 +130,9 @@ class Value(object):
     """Returns the buildbot's equivalent value."""
     raise NotImplementedError()
 
-  def GetBuildbotMeasurementAndTraceNameForPerPageResult(self):
-    measurement, _ = _ConvertValueNameToBuildbotChartAndTraceName(self.name)
-    return measurement, self.page.display_name
+  def GetChartAndTraceNameForPerPageResult(self):
+    chart_name, _ = _ConvertValueNameToChartAndTraceName(self.name)
+    return chart_name, self.page.display_name
 
   @property
   def name_suffix(self):
@@ -142,14 +142,14 @@ class Value(object):
     else:
       return self.name
 
-  def GetBuildbotMeasurementAndTraceNameForComputedSummaryResult(
+  def GetChartAndTraceNameForComputedSummaryResult(
       self, trace_tag):
-    measurement, bb_trace_name = (
-        _ConvertValueNameToBuildbotChartAndTraceName(self.name))
+    chart_name, trace_name = (
+        _ConvertValueNameToChartAndTraceName(self.name))
     if trace_tag:
-      return measurement, bb_trace_name + trace_tag
+      return chart_name, trace_name + trace_tag
     else:
-      return measurement, bb_trace_name
+      return chart_name, trace_name
 
   def GetRepresentativeNumber(self):
     """Gets a single scalar value that best-represents this value.
@@ -290,12 +290,12 @@ def ValueNameFromTraceAndChartName(trace_name, chart_name=None):
         'empty chart_name since this is used to delimit chart_name.trace_name.')
     return trace_name
 
-def _ConvertValueNameToBuildbotChartAndTraceName(value_name):
-  """Converts a value_name into the buildbot equivalent name pair.
+def _ConvertValueNameToChartAndTraceName(value_name):
+  """Converts a value_name into the equivalent chart-trace name pair.
 
   Buildbot represents values by the measurement name and an optional trace name,
   whereas telemetry represents values with a chart_name.trace_name convention,
-  where chart_name is optional.
+  where chart_name is optional. This convention is also used by chart_json.
 
   This converts from the telemetry convention to the buildbot convention,
   returning a 2-tuple (measurement_name, trace_name).

tools/telemetry/telemetry/value/failure.py

@@ -58,7 +58,7 @@ class FailureValue(value_module.Value):
   def GetBuildbotValue(self):
     return None
 
-  def GetBuildbotMeasurementAndTraceNameForPerPageResult(self):
+  def GetChartAndTraceNameForPerPageResult(self):
     return None
 
   def GetRepresentativeNumber(self):

tools/telemetry/telemetry/value/failure_unittest.py

@@ -36,7 +36,7 @@ class ValueTest(TestBase):
     self.assertIsNone(v.GetBuildbotValue())
     self.assertIsNone(v.GetBuildbotDataType(
         value.COMPUTED_PER_PAGE_SUMMARY_OUTPUT_CONTEXT))
-    self.assertIsNone(v.GetBuildbotMeasurementAndTraceNameForPerPageResult())
+    self.assertIsNone(v.GetChartAndTraceNameForPerPageResult())
     self.assertIsNone(v.GetRepresentativeNumber())
     self.assertIsNone(v.GetRepresentativeString())
 

tools/telemetry/telemetry/value/scalar_unittest.py

@@ -29,7 +29,7 @@ class ValueTest(TestBase):
         value.COMPUTED_PER_PAGE_SUMMARY_OUTPUT_CONTEXT))
     self.assertEquals([3], v.GetBuildbotValue())
     self.assertEquals(('x', page0.display_name),
-                      v.GetBuildbotMeasurementAndTraceNameForPerPageResult())
+                      v.GetChartAndTraceNameForPerPageResult())
 
     v = scalar.ScalarValue(page0, 'x', 'unit', 3, important=False)
     self.assertEquals(

tools/telemetry/telemetry/value/skip.py

@@ -31,7 +31,7 @@ class SkipValue(value_module.Value):
   def GetBuildbotValue(self):
     return None
 
-  def GetBuildbotMeasurementAndTraceNameForPerPageResult(self):
+  def GetChartAndTraceNameForPerPageResult(self):
     return None
 
   def GetRepresentativeNumber(self):

tools/telemetry/telemetry/value/skip_unittest.py

@@ -25,7 +25,7 @@ class ValueTest(TestBase):
     self.assertIsNone(v.GetBuildbotValue())
     self.assertIsNone(v.GetBuildbotDataType(
         value.COMPUTED_PER_PAGE_SUMMARY_OUTPUT_CONTEXT))
-    self.assertIsNone(v.GetBuildbotMeasurementAndTraceNameForPerPageResult())
+    self.assertIsNone(v.GetChartAndTraceNameForPerPageResult())
     self.assertIsNone(v.GetRepresentativeNumber())
     self.assertIsNone(v.GetRepresentativeString())
 

tools/telemetry/telemetry/value/string_unittest.py

@@ -29,7 +29,7 @@ class StringValueTest(TestBase):
         value.COMPUTED_PER_PAGE_SUMMARY_OUTPUT_CONTEXT))
     self.assertEquals(['L1'], v.GetBuildbotValue())
     self.assertEquals(('x', page0.display_name),
-                      v.GetBuildbotMeasurementAndTraceNameForPerPageResult())
+                      v.GetChartAndTraceNameForPerPageResult())
 
     v = string.StringValue(page0, 'x', 'label', 'L1', important=False)
     self.assertEquals(

tools/telemetry/telemetry/value/value_unittest.py

@@ -35,7 +35,7 @@ class ValueForTest(value.Value):
   def GetBuildbotValue(self):
     pass
 
-  def GetBuildbotMeasurementAndTraceNameForComputedSummaryResult(
+  def GetChartAndTraceNameForComputedSummaryResult(
       self, trace_tag):
     pass