Add Value FromDict to Telemetry

This CL adds a method FromDict for converting deserialized JSON dicts to
values.

Review URL: https://codereview.chromium.org/417193005

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@287716 0039d316-1c4b-4281-b951-d872f2087c98

tools/telemetry/telemetry/value/__init__.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 """
@@ -20,6 +20,10 @@ Downstream consumers of test results typically want to group these runs
 together, then compute summary statistics across runs. Value provides the
 Merge* family of methods for this kind of aggregation.
 """
+import os
+
+from telemetry.core import discover
+from telemetry.core import util
 
 # When combining a pair of Values togehter, it is sometimes ambiguous whether
 # the values should be concatenated, or one should be picked as representative.
@@ -161,22 +165,20 @@ class Value(object):
     """
     raise NotImplementedError()
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     """Gets the typename for serialization to JSON using AsDict."""
     raise NotImplementedError()
 
   def AsDict(self):
-    """Gets a representation of this value as a dict for eventual
-    serialization to JSON.
-    """
+    """Pre-serializes a value to a dict for output as JSON."""
     return self._AsDictImpl()
 
   def _AsDictImpl(self):
     d = {
       'name': self.name,
       'type': self.GetJSONTypeName(),
-      'unit': self.units,
+      'units': self.units,
     }
 
     if self.description:
@@ -195,6 +197,81 @@ class Value(object):
     return dict([(k, v) for (k, v) in full_dict.iteritems()
                   if k not in base_dict_keys])
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    """Produces a value from a value dict and a page dict.
+
+    Value dicts are produced by serialization to JSON, and must be accompanied
+    by a dict mapping page IDs to pages, also produced by serialization, in
+    order to be completely deserialized. If deserializing multiple values, use
+    ListOfValuesFromListOfDicts instead.
+
+    value_dict: a dictionary produced by AsDict() on a value subclass.
+    page_dict: a dictionary mapping IDs to page objects.
+    """
+    return Value.ListOfValuesFromListOfDicts([value_dict], page_dict)[0]
+
+  @staticmethod
+  def ListOfValuesFromListOfDicts(value_dicts, page_dict):
+    """Takes a list of value dicts to values.
+
+    Given a list of value dicts produced by AsDict, this method
+    deserializes the dicts given a dict mapping page IDs to pages.
+    This method performs memoization for deserializing a list of values
+    efficiently, where FromDict is meant to handle one-offs.
+
+    values: a list of value dicts produced by AsDict() on a value subclass.
+    page_dict: a dictionary mapping IDs to page objects.
+    """
+    value_dir = os.path.dirname(__file__)
+    value_classes = discover.DiscoverClasses(
+        value_dir, util.GetTelemetryDir(),
+        Value, index_by_class_name=True)
+
+    value_json_types = dict((value_classes[x].GetJSONTypeName(), x) for x in
+        value_classes)
+
+    values = []
+    for value_dict in value_dicts:
+      value_class = value_classes[value_json_types[value_dict['type']]]
+      assert 'FromDict' in value_class.__dict__, \
+             'Subclass doesn\'t override FromDict'
+      values.append(value_class.FromDict(value_dict, page_dict))
+
+    return values
+
+  @staticmethod
+  def GetConstructorKwArgs(value_dict, page_dict):
+    """Produces constructor arguments from a value dict and a page dict.
+
+    Takes a dict parsed from JSON and an index of pages and recovers the
+    keyword arguments to be passed to the constructor for deserializing the
+    dict.
+
+    value_dict: a dictionary produced by AsDict() on a value subclass.
+    page_dict: a dictionary mapping IDs to page objects.
+    """
+    d = {
+      'name': value_dict['name'],
+      'units': value_dict['units']
+    }
+
+    description = value_dict.get('description', None)
+    if description:
+      d['description'] = description
+    else:
+      d['description'] = None
+
+    page_id = value_dict.get('page_id', None)
+    if page_id:
+      d['page'] = page_dict[int(page_id)]
+    else:
+      d['page'] = None
+
+    d['important'] = False
+
+    return d
+
 def ValueNameFromTraceAndChartName(trace_name, chart_name=None):
   """Mangles a trace name plus optional chart name into a standard string.
 

tools/telemetry/telemetry/value/failure.py

@@ -29,10 +29,15 @@ class FailureValue(value_module.Value):
       page: The page where this failure occurs.
       message: A string message describing the failure.
     """
+    exc_info = cls._GetExcInfoFromMessage(message)
+    return FailureValue(page, exc_info)
+
+  @staticmethod
+  def _GetExcInfoFromMessage(message):
     try:
       raise Exception(message)
     except Exception:
-      return FailureValue(page, sys.exc_info())
+      return sys.exc_info()
 
   def __repr__(self):
     if self.page:
@@ -61,8 +66,8 @@ class FailureValue(value_module.Value):
   def GetRepresentativeString(self):
     return None
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'failure'
 
   def AsDict(self):
@@ -70,6 +75,19 @@ class FailureValue(value_module.Value):
     d['value'] = GetStringFromExcInfo(self.exc_info)
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    del kwargs['name']
+    del kwargs['units']
+    important = kwargs.get('important', None)
+    if important != None:
+      del kwargs['important']
+    kwargs['exc_info'] = FailureValue._GetExcInfoFromMessage(
+        value_dict['value'])
+
+    return FailureValue(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert False, 'Should not be called.'

tools/telemetry/telemetry/value/failure_unittest.py

@@ -5,6 +5,7 @@
 import os
 import sys
 import unittest
+import traceback
 
 from telemetry import value
 from telemetry.page import page_set
@@ -22,12 +23,12 @@ class TestBase(unittest.TestCase):
 class ValueTest(TestBase):
   def testName(self):
     v0 = failure.FailureValue.FromMessage(self.pages[0], 'Failure')
-    self.assertEqual('Exception', v0.name)
+    self.assertEquals('Exception', v0.name)
     try:
       raise NotImplementedError()
     except Exception:
       v1 = failure.FailureValue(self.pages[0], sys.exc_info())
-    self.assertEqual('NotImplementedError', v1.name)
+    self.assertEquals('NotImplementedError', v1.name)
 
   def testBuildbotAndRepresentativeValue(self):
     v = failure.FailureValue.FromMessage(self.pages[0], 'Failure')
@@ -42,3 +43,19 @@ class ValueTest(TestBase):
     v = failure.FailureValue.FromMessage(self.pages[0], 'Failure')
     d = v.AsDictWithoutBaseClassEntries()
     self.assertTrue(d['value'].find('Exception: Failure') > -1)
+
+  def testFromDict(self):
+    try:
+      raise Exception('test')
+    except Exception:
+      exc_info = sys.exc_info()
+    d = {
+      'type': 'failure',
+      'name': exc_info[0].__name__,
+      'units': '',
+      'value': ''.join(traceback.format_exception(*exc_info))
+    }
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, failure.FailureValue))
+    self.assertEquals(v.name, 'Exception')

tools/telemetry/telemetry/value/histogram.py

@@ -32,7 +32,8 @@ class HistogramValue(value_module.Value):
     super(HistogramValue, self).__init__(page, name, units, important,
                                          description)
     if raw_value_json:
-      assert raw_value == None, 'Dont specify both raw_value and raw_value_json'
+      assert raw_value == None, \
+             'Don\'t specify both raw_value and raw_value_json'
       raw_value = json.loads(raw_value_json)
     if raw_value:
       assert 'buckets' in raw_value
@@ -89,7 +90,8 @@ class HistogramValue(value_module.Value):
   def GetRepresentativeString(self):
     return self.GetBuildbotValue()
 
-  def GetJSONTypeName(self):
+  @staticmethod
+  def GetJSONTypeName():
     return 'histogram'
 
   def AsDict(self):
@@ -97,6 +99,13 @@ class HistogramValue(value_module.Value):
     d['buckets'] = [b.AsDict() for b in self.buckets]
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    kwargs['raw_value'] = value_dict
+
+    return HistogramValue(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert len(values) > 0

tools/telemetry/telemetry/value/histogram_unittest.py

@@ -53,7 +53,7 @@ class ValueTest(TestBase):
           'count': 78
         })
 
-  def testAsDictIsAccurate(self):
+  def testAsDict(self):
     histogram = histogram_module.HistogramValue(
         None, 'x', 'counts',
         raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
@@ -63,3 +63,17 @@ class ValueTest(TestBase):
     self.assertEquals(['buckets'], d.keys())
     self.assertTrue(isinstance(d['buckets'], list))
     self.assertEquals(len(d['buckets']), 1)
+
+  def testFromDict(self):
+    d = {
+      'type': 'histogram',
+      'name': 'x',
+      'units': 'counts',
+      'buckets': [{'low': 1, 'high': 2, 'count': 1}]
+    }
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, histogram_module.HistogramValue))
+    self.assertEquals(
+      ['{"buckets": [{"low": 1, "high": 2, "count": 1}]}'],
+      v.GetBuildbotValue())

tools/telemetry/telemetry/value/list_of_scalar_values.py

@@ -59,8 +59,8 @@ class ListOfScalarValues(value_module.Value):
     return (super(ListOfScalarValues, self).IsMergableWith(that) and
             self.same_page_merge_policy == that.same_page_merge_policy)
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'list_of_scalar_values'
 
   def AsDict(self):
@@ -68,6 +68,13 @@ class ListOfScalarValues(value_module.Value):
     d['values'] = self.values
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    kwargs['values'] = value_dict['values']
+
+    return ListOfScalarValues(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert len(values) > 0

tools/telemetry/telemetry/value/list_of_scalar_values_unittest.py

@@ -77,7 +77,7 @@ class ValueTest(TestBase):
     self.assertEquals(True, vM.important)
     self.assertEquals([1, 2, 3, 4], vM.values)
 
-  def testAsDictIsAccurate(self):
+  def testAsDict(self):
     v = list_of_scalar_values.ListOfScalarValues(
         None, 'x', 'unit', [1, 2],
         same_page_merge_policy=value.PICK_FIRST, important=False)
@@ -86,3 +86,27 @@ class ValueTest(TestBase):
     self.assertEquals(d, {
           'values': [1, 2]
         })
+
+  def testFromDictInts(self):
+    d = {
+      'type': 'list_of_scalar_values',
+      'name': 'x',
+      'units': 'unit',
+      'values': [1, 2]
+    }
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, list_of_scalar_values.ListOfScalarValues))
+    self.assertEquals(v.values, [1, 2])
+
+  def testFromDictFloats(self):
+    d = {
+      'type': 'list_of_scalar_values',
+      'name': 'x',
+      'units': 'unit',
+      'values': [1.3, 2.7]
+    }
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, list_of_scalar_values.ListOfScalarValues))
+    self.assertEquals(v.values, [1.3, 2.7])

tools/telemetry/telemetry/value/list_of_string_values.py

@@ -54,8 +54,8 @@ class ListOfStringValues(value_module.Value):
     return (super(ListOfStringValues, self).IsMergableWith(that) and
             self.same_page_merge_policy == that.same_page_merge_policy)
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'list_of_string_values'
 
   def AsDict(self):
@@ -63,6 +63,13 @@ class ListOfStringValues(value_module.Value):
     d['values'] = self.values
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    kwargs['values'] = value_dict['values']
+
+    return ListOfStringValues(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert len(values) > 0

tools/telemetry/telemetry/value/list_of_string_values_unittest.py

@@ -77,7 +77,7 @@ class ListOfStringValuesTest(TestBase):
     self.assertEquals(True, vM.important)
     self.assertEquals(['L1', 'L2', 'L3', 'L4'], vM.values)
 
-  def testAsDictIsAccurate(self):
+  def testAsDict(self):
     v = list_of_string_values.ListOfStringValues(
         None, 'x', 'unit', ['foo', 'bar'],
         same_page_merge_policy=value.PICK_FIRST, important=False)
@@ -86,3 +86,15 @@ class ListOfStringValuesTest(TestBase):
     self.assertEquals(d, {
           'values': ['foo', 'bar']
         })
+
+  def testFromDict(self):
+    d = {
+      'type': 'list_of_string_values',
+      'name': 'x',
+      'units': 'unit',
+      'values': ['foo', 'bar']
+    }
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, list_of_string_values.ListOfStringValues))
+    self.assertEquals(v.values, ['foo', 'bar'])

tools/telemetry/telemetry/value/scalar.py

@@ -49,8 +49,8 @@ class ScalarValue(value_module.Value):
   def GetRepresentativeString(self):
     return str(self.value)
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'scalar'
 
   def AsDict(self):
@@ -58,6 +58,13 @@ class ScalarValue(value_module.Value):
     d['value'] = self.value
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    kwargs['value'] = value_dict['value']
+
+    return ScalarValue(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert len(values) > 0

tools/telemetry/telemetry/value/scalar_unittest.py

@@ -60,10 +60,36 @@ class ValueTest(TestBase):
     self.assertEquals(True, vM.important)
     self.assertEquals([1, 2], vM.values)
 
-  def testAsDictIsAccurate(self):
+  def testAsDict(self):
     v = scalar.ScalarValue(None, 'x', 'unit', 42, important=False)
     d = v.AsDictWithoutBaseClassEntries()
 
     self.assertEquals(d, {
           'value': 42
         })
+
+  def testFromDictInt(self):
+    d = {
+      'type': 'scalar',
+      'name': 'x',
+      'units': 'unit',
+      'value': 42
+    }
+
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, scalar.ScalarValue))
+    self.assertEquals(v.value, 42)
+
+  def testFromDictFloat(self):
+    d = {
+      'type': 'scalar',
+      'name': 'x',
+      'units': 'unit',
+      'value': 42.4
+    }
+
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, scalar.ScalarValue))
+    self.assertEquals(v.value, 42.4)

tools/telemetry/telemetry/value/skip.py

@@ -39,8 +39,8 @@ class SkipValue(value_module.Value):
   def GetRepresentativeString(self):
     return None
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'skip'
 
   def AsDict(self):
@@ -48,6 +48,18 @@ class SkipValue(value_module.Value):
     d['reason'] = self._reason
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    del kwargs['name']
+    del kwargs['units']
+    important = kwargs.get('important', None)
+    if important != None:
+      del kwargs['important']
+    kwargs['reason'] = value_dict['reason']
+
+    return SkipValue(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert False, 'Should not be called.'

tools/telemetry/telemetry/value/skip_unittest.py

@@ -32,3 +32,14 @@ class ValueTest(TestBase):
     v = skip.SkipValue(self.pages[0], 'page skipped for testing reason')
     d = v.AsDictWithoutBaseClassEntries()
     self.assertEquals(d['reason'], 'page skipped for testing reason')
+
+  def testFromDict(self):
+    d = {
+      'type': 'skip',
+      'name': 'skip',
+      'units': '',
+      'reason': 'page skipped for testing reason'
+    }
+    v = value.Value.FromDict(d, {})
+    self.assertTrue(isinstance(v, skip.SkipValue))
+    self.assertEquals(v.reason, 'page skipped for testing reason')

tools/telemetry/telemetry/value/string.py

@@ -47,8 +47,8 @@ class StringValue(value_module.Value):
   def GetRepresentativeString(self):
     return str(self.value)
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'string'
 
   def AsDict(self):
@@ -56,6 +56,13 @@ class StringValue(value_module.Value):
     d['value'] = self.value
     return d
 
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value_module.Value.GetConstructorKwArgs(value_dict, page_dict)
+    kwargs['value'] = value_dict['value']
+
+    return StringValue(**kwargs)
+
   @classmethod
   def MergeLikeValuesFromSamePage(cls, values):
     assert len(values) > 0

tools/telemetry/telemetry/value/string_unittest.py

@@ -60,10 +60,23 @@ class StringValueTest(TestBase):
     self.assertEquals(True, vM.important)
     self.assertEquals(['L1', 'L2'], vM.values)
 
-  def testAsDictIsAccurate(self):
+  def testAsDict(self):
     v = string.StringValue(None, 'x', 'unit', 'foo', important=False)
     d = v.AsDictWithoutBaseClassEntries()
 
     self.assertEquals(d, {
           'value': 'foo'
         })
+
+  def testFromDict(self):
+    d = {
+      'type': 'string',
+      'name': 'x',
+      'units': 'unit',
+      'value': 'foo'
+    }
+
+    v = value.Value.FromDict(d, {})
+
+    self.assertTrue(isinstance(v, string.StringValue))
+    self.assertEquals(v.value, 'foo')

tools/telemetry/telemetry/value/value_unittest.py

@@ -44,15 +44,25 @@ class ValueForTest(value.Value):
   def GetRepresentativeString(self):
     pass
 
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     pass
 
 class ValueForAsDictTest(ValueForTest):
-  @classmethod
-  def GetJSONTypeName(cls):
+  @staticmethod
+  def GetJSONTypeName():
     return 'baz'
 
+class ValueForFromDictTest(ValueForTest):
+  @staticmethod
+  def FromDict(value_dict, page_dict):
+    kwargs = value.Value.GetConstructorKwArgs(value_dict, page_dict)
+    return ValueForFromDictTest(**kwargs)
+
+  @staticmethod
+  def GetJSONTypeName():
+    return 'value_for_from_dict_test'
+
 class ValueTest(TestBase):
   def testCompat(self):
     page0 = self.pages[0]
@@ -85,7 +95,7 @@ class ValueTest(TestBase):
     self.assertEquals(d, {
           'name': 'x',
           'type': 'baz',
-          'unit': 'unit',
+          'units': 'unit',
         })
 
   def testAsDictWithPage(self):
@@ -112,3 +122,77 @@ class ValueTest(TestBase):
   def testAsDictWithoutDescription(self):
     v = ValueForAsDictTest(None, 'x', 'unit', important=False, description=None)
     self.assertNotIn('description', v.AsDict())
+
+  def testFromDictBaseKeys(self):
+    d = {
+      'type': 'value_for_from_dict_test',
+      'name': 'x',
+      'units': 'unit'
+    }
+
+    v = value.Value.FromDict(d, None)
+    self.assertEquals(v.name, 'x')
+    self.assertTrue(isinstance(v, ValueForFromDictTest))
+    self.assertEquals(v.units, 'unit')
+
+  def testFromDictWithPage(self):
+    page0 = self.pages[0]
+    page_dict = {page0.id: page0}
+
+    d = {
+      'type': 'value_for_from_dict_test',
+      'name': 'x',
+      'units': 'unit',
+      'page_id': page0.id
+    }
+
+    v = value.Value.FromDict(d, page_dict)
+
+    self.assertEquals(v.page.id, page0.id)
+
+  def testFromDictWithoutPage(self):
+    d = {
+      'type': 'value_for_from_dict_test',
+      'name': 'x',
+      'units': 'unit'
+    }
+
+    v = value.Value.FromDict(d, {})
+
+    self.assertEquals(v.page, None)
+
+  def testFromDictWithDescription(self):
+    d = {
+          'type': 'value_for_from_dict_test',
+          'name': 'x',
+          'units': 'unit',
+          'description': 'foo'
+        }
+
+    v = value.Value.FromDict(d, {})
+    self.assertEquals(v.description, 'foo')
+
+  def testFromDictWithoutDescription(self):
+    d = {
+          'type': 'value_for_from_dict_test',
+          'name': 'x',
+          'units': 'unit'
+        }
+
+    v = value.Value.FromDict(d, {})
+    self.assertEquals(v.description, None)
+
+  def testListOfValuesFromListOfDicts(self):
+    d0 = {
+          'type': 'value_for_from_dict_test',
+          'name': 'x',
+          'units': 'unit'
+        }
+    d1 = {
+          'type': 'value_for_from_dict_test',
+          'name': 'y',
+          'units': 'unit'
+        }
+    vs = value.Value.ListOfValuesFromListOfDicts([d0, d1], {})
+    self.assertEquals(vs[0].name, 'x')
+    self.assertEquals(vs[1].name, 'y')