Telemetry / Memory benchmark fix: Separate histograms for different tests.

We can't assume that all histograms are reset between test cases. We need to
remember start values for histograms before the test run, and subtract them from the
results.

memory_benchmark results before and after this CL are not comparable.

R=nduca
BUG=NONE
NOTRY=true


Review URL: https://chromiumcodereview.appspot.com/12221137

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@182467 0039d316-1c4b-4281-b951-d872f2087c98

tools/perf/perf_tools/histogram.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import json
+import logging
+
+def SubtractHistogram(histogram_json, start_histogram_json):
+  """Subtracts a previous histogram from a histogram. Both parameters are json
+  serializations of histograms."""
+  start_histogram = json.loads(start_histogram_json)
+  # It's ok if the start histogram is empty (we had no data, maybe even no
+  # histogram at all, at the start of the test).
+  if 'buckets' not in start_histogram:
+    return histogram_json
+
+  start_histogram_buckets = dict()
+  for b in start_histogram['buckets']:
+    start_histogram_buckets[b['low']] = b['count']
+
+  new_buckets = []
+  histogram = json.loads(histogram_json)
+  for b in histogram['buckets']:
+    new_bucket = b
+    low = b['low']
+    if low in start_histogram_buckets:
+      new_bucket['count'] = b['count'] - start_histogram_buckets[low]
+      if new_bucket['count'] < 0:
+        logging.error('Histogram subtraction error, starting histogram most '
+                      'probably invalid.')
+    if new_bucket['count']:
+      new_buckets.append(new_bucket)
+  histogram['buckets'] = new_buckets
+  histogram['count'] -= start_histogram['count']
+
+  return json.dumps(histogram)

tools/perf/perf_tools/histogram_measurement.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+from perf_tools import histogram as histogram_module
+
+BROWSER_HISTOGRAM = 'browser_histogram'
+RENDERER_HISTOGRAM = 'renderer_histogram'
+
+class HistogramMeasurement(object):
+  def __init__(self, histogram, histogram_type):
+    self.name = histogram['name']
+    self.units = histogram['units']
+    self.histogram_type = histogram_type
+    self._start_values = dict()
+
+  def Start(self, page, tab):
+    """Get the starting value for the histogram. This value will then be
+    subtracted from the actual measurement."""
+    data = self._GetHistogramFromDomAutomation(tab)
+    if data:
+      self._start_values[page.url + self.name] = data
+
+  def GetValue(self, page, tab, results):
+    data = self._GetHistogramFromDomAutomation(tab)
+    if not data:
+      return
+    new_histogram = histogram_module.SubtractHistogram(
+        data, self._start_values[page.url + self.name])
+    results.Add(self.name.replace('.', '_'), self.units,
+                new_histogram, data_type='histogram')
+
+  @property
+  def histogram_function(self):
+    if self.histogram_type == BROWSER_HISTOGRAM:
+      return 'getBrowserHistogram'
+    return 'getHistogram'
+
+  def _GetHistogramFromDomAutomation(self, tab):
+    js = ('window.domAutomationController.%s ? '
+          'window.domAutomationController.%s("%s") : ""' %
+          (self.histogram_function, self.histogram_function, self.name))
+    return tab.EvaluateJavaScript(js)

tools/perf/perf_tools/histogram_unittest.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+import json
+import unittest
+
+from perf_tools import histogram as histogram_module
+
+class TestHistogram(unittest.TestCase):
+  def testSubtractHistogram(self):
+    baseline_histogram = """{"count": 3, "buckets": [
+{"low": 1, "high": 2, "count": 1},
+{"low": 2, "high": 3, "count": 2}]}"""
+
+    histogram = """{"count": 14, "buckets": [
+{"low": 1, "high": 2, "count": 1},
+{"low": 2, "high": 3, "count": 3},
+{"low": 3, "high": 4, "count": 10}]}"""
+
+    new_histogram = json.loads(
+        histogram_module.SubtractHistogram(histogram, baseline_histogram))
+    new_buckets = dict()
+    for b in new_histogram['buckets']:
+      new_buckets[b['low']] = b['count']
+    self.assertFalse(1 in new_buckets)
+    self.assertEquals(1, new_buckets[2])
+    self.assertEquals(10, new_buckets[3])

tools/perf/perf_tools/memory_benchmark.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
+from perf_tools import histogram_measurement
 from telemetry import multi_page_benchmark
 
 MEMORY_HISTOGRAMS = [
@@ -15,6 +16,17 @@ BROWSER_MEMORY_HISTOGRAMS =  [
 class MemoryBenchmark(multi_page_benchmark.MultiPageBenchmark):
   def __init__(self):
     super(MemoryBenchmark, self).__init__('stress_memory')
+    self.histograms = (
+        [histogram_measurement.HistogramMeasurement(
+            h, histogram_measurement.RENDERER_HISTOGRAM)
+         for h in MEMORY_HISTOGRAMS] +
+        [histogram_measurement.HistogramMeasurement(
+            h, histogram_measurement.BROWSER_HISTOGRAM)
+         for h in BROWSER_MEMORY_HISTOGRAMS])
+
+  def DidNavigateToPage(self, page, tab):
+    for h in self.histograms:
+      h.Start(page, tab)
 
   def CustomizeBrowserOptions(self, options):
     options.AppendExtraBrowserArg('--dom-automation')
@@ -31,18 +43,5 @@ class MemoryBenchmark(multi_page_benchmark.MultiPageBenchmark):
     return hasattr(page, 'stress_memory')
 
   def MeasurePage(self, page, tab, results):
-    for histogram in MEMORY_HISTOGRAMS:
-      self._GetHistogramFromDomAutomation(tab, 'getHistogram', histogram,
-                                          results)
-    for histogram in BROWSER_MEMORY_HISTOGRAMS:
-      self._GetHistogramFromDomAutomation(tab, 'getBrowserHistogram', histogram,
-                                          results)
-
-  def _GetHistogramFromDomAutomation(self, tab, func, histogram, results):
-    name = histogram['name']
-    js = ('window.domAutomationController.%s ? '
-          'window.domAutomationController.%s("%s") : ""' % (func, func, name))
-    data = tab.EvaluateJavaScript(js)
-    if data:
-      results.Add(name.replace('.', '_'), histogram['units'], data,
-                  data_type='histogram')
+    for h in self.histograms:
+      h.GetValue(page, tab, results)