Skip to content

GitLab

T
tangled
Commit a3dd9802 authored by Emily Hanley's avatar Emily Hanley Committed by Commit Bot
Browse files
Options

Implement press benchmark harness. 

This harness contains the APIs necessary for all press benchmarks, but
they will remain separate benchmarks.  There is just now common
functionality that they can gain from this harness.

This CL includes:
1) New press benchmark that enables both in test javascript metrics
as well as tracing metrics
2) New press pages that implements the new measurement
3) Converted speedometer, speedometer2, and dromaeo benchmarks

Bug: 714231
Change-Id: I487612d38f06ecd8a4c8905102972b134497d0c0
Reviewed-on: https://chromium-review.googlesource.com/c/1299739
Commit-Queue: Emily Hanley <eyaich@chromium.org>
Reviewed-by: default avatarCaleb Rouleau <crouleau@chromium.org>
Reviewed-by: default avatarJuan Antonio Navarro Pérez <perezju@chromium.org>
Reviewed-by: default avatarNed Nguyen <nednguyen@google.com>
Cr-Commit-Position: refs/heads/master@{#605349}
parent b957a0c4
Hide whitespace changes
Inline Side-by-side
Showing with 457 additions and 301 deletions
tools/perf/benchmarks/dromaeo.py
View file @ a3dd9802
...@@ -2,109 +2,23 @@ ...@@ -2,109 +2,23 @@
# Use of this source code is governed by a BSD-style license that can be # Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. # found in the LICENSE file.
import json from benchmarks import press
import math
import os
from core import perf_benchmark
from telemetry import benchmark from telemetry import benchmark
from telemetry import page as page_module
from telemetry.page import legacy_page_test
from telemetry import story
from telemetry.value import scalar
class _DromaeoMeasurement(legacy_page_test.LegacyPageTest):
def __init__(self):
super(_DromaeoMeasurement, self).__init__()
def ValidateAndMeasurePage(self, page, tab, results):
tab.WaitForJavaScriptCondition(
'window.document.getElementById("pause") &&' +
'window.document.getElementById("pause").value == "Run"',
timeout=120)
# Start spying on POST request that will report benchmark results, and
# intercept result data.
tab.ExecuteJavaScript("""
(function() {
var real_jquery_ajax_ = window.jQuery;
window.results_ = "";
window.jQuery.ajax = function(request) {
if (request.url == "store.php") {
window.results_ = decodeURIComponent(request.data);
window.results_ = window.results_.substring(
window.results_.indexOf("=") + 1,
window.results_.lastIndexOf("&"));
real_jquery_ajax_(request);
}
};
})();""")
# Starts benchmark.
tab.ExecuteJavaScript('window.document.getElementById("pause").click();')
tab.WaitForJavaScriptCondition('!!window.results_', timeout=600)
score = json.loads(tab.EvaluateJavaScript('window.results_ || "[]"'))
def Escape(k): from page_sets import dromaeo_pages
chars = [' ', '.', '-', '/', '(', ')', '*']
for c in chars:
k = k.replace(c, '_')
return k
def AggregateData(container, key, value):
if key not in container:
container[key] = {'count': 0, 'sum': 0}
container[key]['count'] += 1
container[key]['sum'] += math.log(value)
suffix = page.url[page.url.index('?') + 1:]
def AddResult(name, value):
important = False
if name == suffix:
important = True
results.AddValue(scalar.ScalarValue(
results.current_page, Escape(name), 'runs/s', value, important))
aggregated = {}
for data in score:
AddResult('%s/%s' % (data['collection'], data['name']),
data['mean'])
top_name = data['collection'].split('-', 1)[0]
AggregateData(aggregated, top_name, data['mean'])
collection_name = data['collection']
AggregateData(aggregated, collection_name, data['mean'])
for key, value in aggregated.iteritems():
AddResult(key, math.exp(value['sum'] / value['count']))
@benchmark.Info(component='Blink>Bindings', @benchmark.Info(component='Blink>Bindings',
emails=['jbroman@chromium.org', emails=['jbroman@chromium.org',
'yukishiino@chromium.org', 'yukishiino@chromium.org',
'haraken@chromium.org']) 'haraken@chromium.org'])
class DromaeoBenchmark(perf_benchmark.PerfBenchmark): # pylint: disable=protected-access
class DromaeoBenchmark(press._PressBenchmark):
test = _DromaeoMeasurement
@classmethod @classmethod
def Name(cls): def Name(cls):
return 'dromaeo' return 'dromaeo'
def CreateStorySet(self, options): def CreateStorySet(self, options):
archive_data_file = '../page_sets/data/dromaeo.json' return dromaeo_pages.DromaeoStorySet()
ps = story.StorySet(
archive_data_file=archive_data_file,
base_dir=os.path.dirname(os.path.abspath(__file__)),
cloud_storage_bucket=story.PUBLIC_BUCKET)
for query_param in ['dom-attr', 'dom-modify', 'dom-query', 'dom-traverse']:
url = 'http://dromaeo.com?%s' % query_param
ps.AddStory(page_module.Page(
url, ps, ps.base_dir, make_javascript_deterministic=False, name=url))
return ps
tools/perf/benchmarks/press.py 0 → 100644
View file @ a3dd9802
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Base class for a PressBenchmark.
This benchmark manages both PressStory objects that
implement javascript based metrics as well as can
compute TMBv2 metrics.
Example implementation:
FooPressBenchmark(press._PressBenchmark):
@classmethod
def Name(clas):
return Foo;
def CreateStorySet():
// Return a set of stories inheriting from
// page_sets.PressStory
def CreateCoreTimelineBasedMeasurementOptions()
// Implement to define tracing metrics you
// want on top of any javascript metrics
// implemented in your stories
"""
from core import perf_benchmark
from measurements import dual_metric_measurement
class _PressBenchmark(perf_benchmark.PerfBenchmark):
test = dual_metric_measurement.DualMetricMeasurement
tools/perf/benchmarks/speedometer.py
View file @ a3dd9802
...@@ -15,100 +15,20 @@ popular websites in the world, such as Facebook and Twitter. The performance of ...@@ -15,100 +15,20 @@ popular websites in the world, such as Facebook and Twitter. The performance of
these types of operations depends on the speed of the DOM APIs, the JavaScript these types of operations depends on the speed of the DOM APIs, the JavaScript
engine, CSS style resolution, layout, and other technologies. engine, CSS style resolution, layout, and other technologies.
""" """
import os
from core import perf_benchmark
from telemetry import benchmark from telemetry import benchmark
from telemetry import page as page_module
from telemetry.page import legacy_page_test
from telemetry import story
from telemetry.value import list_of_scalar_values
class SpeedometerMeasurement(legacy_page_test.LegacyPageTest):
enabled_suites = [
'VanillaJS-TodoMVC',
'EmberJS-TodoMVC',
'BackboneJS-TodoMVC',
'jQuery-TodoMVC',
'AngularJS-TodoMVC',
'React-TodoMVC',
'FlightJS-TodoMVC'
]
def __init__(self):
super(SpeedometerMeasurement, self).__init__()
def ValidateAndMeasurePage(self, page, tab, results):
tab.WaitForDocumentReadyStateToBeComplete()
iterationCount = 10
# A single iteration on android takes ~75 seconds, the benchmark times out
# when running for 10 iterations.
if tab.browser.platform.GetOSName() == 'android':
iterationCount = 3
tab.ExecuteJavaScript("""
// Store all the results in the benchmarkClient
benchmarkClient._measuredValues = []
benchmarkClient.didRunSuites = function(measuredValues) {
benchmarkClient._measuredValues.push(measuredValues);
benchmarkClient._timeValues.push(measuredValues.total);
};
benchmarkClient.iterationCount = {{ count }};
startTest();
""",
count=iterationCount)
tab.WaitForJavaScriptCondition(
'benchmarkClient._finishedTestCount == benchmarkClient.testsCount',
timeout=600)
results.AddValue(list_of_scalar_values.ListOfScalarValues(
page, 'Total', 'ms',
tab.EvaluateJavaScript('benchmarkClient._timeValues'),
important=True))
results.AddValue(list_of_scalar_values.ListOfScalarValues(
page, 'RunsPerMinute', 'score',
tab.EvaluateJavaScript(
'[parseFloat(document.getElementById("result-number").innerText)];'
),
important=True))
# Extract the timings for each suite
for suite_name in self.enabled_suites:
results.AddValue(list_of_scalar_values.ListOfScalarValues(
page, suite_name, 'ms',
tab.EvaluateJavaScript("""
var suite_times = [];
for(var i = 0; i < benchmarkClient.iterationCount; i++) {
suite_times.push(
benchmarkClient._measuredValues[i].tests[{{ key }}].total);
};
suite_times;
""",
key=suite_name), important=False))
import page_sets
from benchmarks import press
@benchmark.Info(emails=['hablich@chromium.org'], @benchmark.Info(emails=['hablich@chromium.org'],
component='Blink') component='Blink')
class Speedometer(perf_benchmark.PerfBenchmark): class Speedometer(press._PressBenchmark): # pylint: disable=protected-access
test = SpeedometerMeasurement
@classmethod @classmethod
def Name(cls): def Name(cls):
return 'speedometer' return 'speedometer'
def CreateStorySet(self, options): def CreateStorySet(self, options):
ps = story.StorySet( return page_sets.SpeedometerStorySet()
base_dir=os.path.dirname(os.path.abspath(__file__)),
archive_data_file='../page_sets/data/speedometer.json',
cloud_storage_bucket=story.PUBLIC_BUCKET)
ps.AddStory(page_module.Page(
'http://browserbench.org/Speedometer/', ps, ps.base_dir,
make_javascript_deterministic=False,
name='http://browserbench.org/Speedometer/'))
return ps
@benchmark.Info(emails=['hablich@chromium.org'], @benchmark.Info(emails=['hablich@chromium.org'],
component='Blink') component='Blink')
...@@ -123,4 +43,5 @@ class V8SpeedometerFuture(Speedometer): ...@@ -123,4 +43,5 @@ class V8SpeedometerFuture(Speedometer):
return 'speedometer-future' return 'speedometer-future'
def SetExtraBrowserOptions(self, options): def SetExtraBrowserOptions(self, options):
super(V8SpeedometerFuture, self).SetExtraBrowserOptions(options)
options.AppendExtraBrowserArgs('--enable-features=V8VmFuture') options.AppendExtraBrowserArgs('--enable-features=V8VmFuture')
tools/perf/benchmarks/speedometer2.py
View file @ a3dd9802
...@@ -8,121 +8,21 @@ ...@@ -8,121 +8,21 @@
import os import os
import re import re
from benchmarks import press
from core import path_util from core import path_util
from core import perf_benchmark
from telemetry import benchmark from telemetry import benchmark
from telemetry import page as page_module
from telemetry.page import legacy_page_test
from telemetry import story from telemetry import story
from telemetry.value import list_of_scalar_values
from page_sets import speedometer2_pages
_SPEEDOMETER_DIR = os.path.join(path_util.GetChromiumSrcDir(), _SPEEDOMETER_DIR = os.path.join(path_util.GetChromiumSrcDir(),
'third_party', 'blink', 'perf_tests', 'speedometer') 'third_party', 'blink', 'perf_tests', 'speedometer')
_SPEEDOMETER_SUITE_NAME_BASE = '{0}-TodoMVC'
_SPEEDOMETER_SUITES = [
'VanillaJS',
'Vanilla-ES2015',
'Vanilla-ES2015-Babel-Webpack',
'React',
'React-Redux',
'EmberJS',
'EmberJS-Debug',
'BackboneJS',
'AngularJS',
'Angular2-TypeScript',
'VueJS',
'jQuery',
'Preact',
'Inferno',
'Elm',
'Flight'
]
class Speedometer2Measurement(legacy_page_test.LegacyPageTest):
def __init__(self, should_filter_suites, filtered_suite_names=None,
enable_smoke_test_mode=False):
super(Speedometer2Measurement, self).__init__()
self.should_filter_suites_ = should_filter_suites
self.filtered_suites_ = filtered_suite_names
self.enable_smoke_test_mode = enable_smoke_test_mode
def ValidateAndMeasurePage(self, page, tab, results):
tab.WaitForDocumentReadyStateToBeComplete()
iterationCount = 10
# A single iteration on android takes ~75 seconds, the benchmark times out
# when running for 10 iterations.
if tab.browser.platform.GetOSName() == 'android':
iterationCount = 3
# For a smoke test one iteration is sufficient
if self.enable_smoke_test_mode:
iterationCount = 1
if self.should_filter_suites_:
tab.ExecuteJavaScript("""
Suites.forEach(function(suite) {
suite.disabled = {{ filtered_suites }}.indexOf(suite.name) < 0;
});
""", filtered_suites=self.filtered_suites_)
enabled_suites = tab.EvaluateJavaScript("""
(function() {
var suitesNames = [];
Suites.forEach(function(s) {
if (!s.disabled)
suitesNames.push(s.name);
});
return suitesNames;
})();""")
tab.ExecuteJavaScript("""
// Store all the results in the benchmarkClient
var testDone = false;
var iterationCount = {{ count }};
var benchmarkClient = {};
var suiteValues = [];
benchmarkClient.didRunSuites = function(measuredValues) {
suiteValues.push(measuredValues);
};
benchmarkClient.didFinishLastIteration = function () {
testDone = true;
};
var runner = new BenchmarkRunner(Suites, benchmarkClient);
runner.runMultipleIterations(iterationCount);
""",
count=iterationCount)
tab.WaitForJavaScriptCondition('testDone', timeout=600)
if not self.should_filter_suites_:
results.AddValue(list_of_scalar_values.ListOfScalarValues(
page, 'Total', 'ms',
tab.EvaluateJavaScript('suiteValues.map(each => each.total)'),
important=True))
results.AddValue(list_of_scalar_values.ListOfScalarValues(
page, 'RunsPerMinute', 'score',
tab.EvaluateJavaScript('suiteValues.map(each => each.score)'),
important=True))
# Extract the timings for each suite
for suite_name in enabled_suites:
results.AddValue(list_of_scalar_values.ListOfScalarValues(
page, suite_name, 'ms',
tab.EvaluateJavaScript("""
var suite_times = [];
for(var i = 0; i < iterationCount; i++) {
suite_times.push(
suiteValues[i].tests[{{ key }}].total);
};
suite_times;
""",
key=suite_name), important=False))
@benchmark.Info(emails=['hablich@chromium.org'], @benchmark.Info(emails=['hablich@chromium.org'],
component='Blink') component='Blink')
class Speedometer2(perf_benchmark.PerfBenchmark): class Speedometer2(press._PressBenchmark): # pylint: disable=protected-access
"""Speedometer2 Benchmark. """Speedometer2 Benchmark.
Runs all the speedometer 2 suites by default. Add --suite=<regex> to filter Runs all the speedometer 2 suites by default. Add --suite=<regex> to filter
...@@ -136,29 +36,15 @@ class Speedometer2(perf_benchmark.PerfBenchmark): ...@@ -136,29 +36,15 @@ class Speedometer2(perf_benchmark.PerfBenchmark):
def Name(cls): def Name(cls):
return 'speedometer2' return 'speedometer2'
@staticmethod
def GetFullSuiteName(name):
return _SPEEDOMETER_SUITE_NAME_BASE.format(name)
@staticmethod
def GetSuites(suite_regex):
if not suite_regex:
return []
exp = re.compile(suite_regex)
return [name for name in _SPEEDOMETER_SUITES
if exp.search(Speedometer2.GetFullSuiteName(name))]
def CreatePageTest(self, options):
should_filter_suites = bool(options.suite)
filtered_suite_names = map(Speedometer2.GetFullSuiteName,
Speedometer2.GetSuites(options.suite))
return Speedometer2Measurement(should_filter_suites, filtered_suite_names,
self.enable_smoke_test_mode)
def CreateStorySet(self, options): def CreateStorySet(self, options):
should_filter_suites = bool(options.suite)
filtered_suite_names = map(
speedometer2_pages.Speedometer2Story.GetFullSuiteName,
speedometer2_pages.Speedometer2Story.GetSuites(options.suite))
ps = story.StorySet(base_dir=_SPEEDOMETER_DIR) ps = story.StorySet(base_dir=_SPEEDOMETER_DIR)
ps.AddStory(page_module.Page( ps.AddStory(speedometer2_pages.Speedometer2Story(ps, should_filter_suites,
'file://InteractiveRunner.html', ps, ps.base_dir, name='Speedometer2')) filtered_suite_names, self.enable_smoke_test_mode))
return ps return ps
@classmethod @classmethod
...@@ -170,7 +56,7 @@ class Speedometer2(perf_benchmark.PerfBenchmark): ...@@ -170,7 +56,7 @@ class Speedometer2(perf_benchmark.PerfBenchmark):
def ProcessCommandLineArgs(cls, parser, args): def ProcessCommandLineArgs(cls, parser, args):
if args.suite: if args.suite:
try: try:
if not Speedometer2.GetSuites(args.suite): if not speedometer2_pages.Speedometer2Story.GetSuites(args.suite):
raise parser.error('--suite: No matches.') raise parser.error('--suite: No matches.')
except re.error: except re.error:
raise parser.error('--suite: Invalid regex.') raise parser.error('--suite: Invalid regex.')
......
tools/perf/measurements/dual_metric_measurement.py 0 → 100644
View file @ a3dd9802
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.web_perf import story_test
from telemetry.web_perf import timeline_based_measurement
class DualMetricMeasurement(story_test.StoryTest):
"""Test class for a benchmark that aggregates all metrics.
Assumes both javascript as well as tracing metrics might be defined.
All pages associated with this measurement must implement
GetJavascriptMetricValues()
"""
def __init__(self, tbm_options):
super(DualMetricMeasurement, self).__init__()
# Only enable tracing if metrics have been specified.
if tbm_options.GetTimelineBasedMetrics():
self._tbm_test = timeline_based_measurement.TimelineBasedMeasurement(
tbm_options)
self._enable_tracing = True
else:
self._enable_tracing = False
def WillRunStory(self, platform):
if self._enable_tracing:
self._tbm_test.WillRunStory(platform)
def Measure(self, platform, results):
for value in results.current_page.GetJavascriptMetricValues():
results.AddValue(value)
# This call is necessary to convert the current ScalarValues to
# histograms before more histograms are added. If we don't,
# when histograms get added by TBM2 page_test_results will see those and
# not convert any existing values because it assumes they are already
# converted. Therefore, so the javascript metrics don't get dropped, we
# have to convert them first.
results.PopulateHistogramSet()
if self._enable_tracing:
self._tbm_test.Measure(platform, results)
def DidRunStory(self, platform, results):
if self._enable_tracing:
self._tbm_test.DidRunStory(platform, results)
tools/perf/page_sets/dromaeo_pages.py 0 → 100644
View file @ a3dd9802
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import math
from telemetry import story
from telemetry.value import scalar
from page_sets import press_story
class DromaeoStory(press_story.PressStory):
def __init__(self, url, ps):
self.URL = url
super(DromaeoStory, self).__init__(ps)
def ExecuteTest(self, action_runner):
action_runner.WaitForJavaScriptCondition(
'window.document.getElementById("pause") &&' +
'window.document.getElementById("pause").value == "Run"',
timeout=120)
# Start spying on POST request that will report benchmark results, and
# intercept result data.
action_runner.ExecuteJavaScript("""
(function() {
var real_jquery_ajax_ = window.jQuery;
window.results_ = "";
window.jQuery.ajax = function(request) {
if (request.url == "store.php") {
window.results_ = decodeURIComponent(request.data);
window.results_ = window.results_.substring(
window.results_.indexOf("=") + 1,
window.results_.lastIndexOf("&"));
real_jquery_ajax_(request);
}
};
})();""")
# Starts benchmark.
action_runner.ExecuteJavaScript(
'window.document.getElementById("pause").click();')
action_runner.WaitForJavaScriptCondition('!!window.results_', timeout=600)
def ParseTestResults(self, action_runner):
score = json.loads(
action_runner.EvaluateJavaScript('window.results_ || "[]"'))
def Escape(k):
chars = [' ', '.', '-', '/', '(', ')', '*']
for c in chars:
k = k.replace(c, '_')
return k
def AggregateData(container, key, value):
if key not in container:
container[key] = {'count': 0, 'sum': 0}
container[key]['count'] += 1
container[key]['sum'] += math.log(value)
suffix = self.url[self.url.index('?') + 1:]
def AddResult(name, value):
important = False
if name == suffix:
important = True
self.AddJavascriptMetricValue(scalar.ScalarValue(
self, Escape(name), 'runs/s', value, important))
aggregated = {}
for data in score:
AddResult('%s/%s' % (data['collection'], data['name']),
data['mean'])
top_name = data['collection'].split('-', 1)[0]
AggregateData(aggregated, top_name, data['mean'])
collection_name = data['collection']
AggregateData(aggregated, collection_name, data['mean'])
for key, value in aggregated.iteritems():
AddResult(key, math.exp(value['sum'] / value['count']))
class DromaeoStorySet(story.StorySet):
def __init__(self):
super(DromaeoStorySet, self).__init__(
archive_data_file='../page_sets/data/dromaeo.json',
cloud_storage_bucket=story.PUBLIC_BUCKET)
for query_param in ['dom-attr', 'dom-modify', 'dom-query', 'dom-traverse']:
url = 'http://dromaeo.com?%s' % query_param
self.AddStory(DromaeoStory(url, self))
tools/perf/page_sets/press_story.py 0 → 100644
View file @ a3dd9802
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
class PressStory(page_module.Page):
"""Base class for Press stories.
Override ExecuteTest to execute javascript on the page and
ParseTestResults to obtain javascript metrics from page.
Example Implementation:
class FooPressStory:
URL = 'http://foo'
def ExecuteTest(self, action_runner):
//Execute some javascript
def ParseTestResults(self, action_runner):
some_value = action_runner.EvaluateJavascript("some javascript")
self.AddJavascriptMetricValue(some_value)
"""
URL = None
DETERMINISTIC_JS = False
NAME = None
def __init__(self, ps):
super(PressStory, self).__init__(
self.URL, ps,
base_dir=ps.base_dir,
make_javascript_deterministic=self.DETERMINISTIC_JS,
name=self.NAME if self.NAME else self.URL)
self._values = []
def GetJavascriptMetricValues(self):
return self._values
def AddJavascriptMetricValue(self, value):
self._values.append(value)
def ExecuteTest(self, action_runner):
pass
def ParseTestResults(self, action_runner):
pass
def RunPageInteractions(self, action_runner):
self.ExecuteTest(action_runner)
self.ParseTestResults(action_runner)
tools/perf/page_sets/speedometer2_pages.py 0 → 100644
View file @ a3dd9802
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Apple's Speedometer 2 performance benchmark pages
"""
import re
from telemetry.value import list_of_scalar_values
from page_sets import press_story
_SPEEDOMETER_SUITE_NAME_BASE = '{0}-TodoMVC'
_SPEEDOMETER_SUITES = [
'VanillaJS',
'Vanilla-ES2015',
'Vanilla-ES2015-Babel-Webpack',
'React',
'React-Redux',
'EmberJS',
'EmberJS-Debug',
'BackboneJS',
'AngularJS',
'Angular2-TypeScript',
'VueJS',
'jQuery',
'Preact',
'Inferno',
'Elm',
'Flight'
]
class Speedometer2Story(press_story.PressStory):
URL = 'file://InteractiveRunner.html'
NAME = 'Speedometer2'
def __init__(self, ps, should_filter_suites, filtered_suite_names=None,
enable_smoke_test_mode=False):
super(Speedometer2Story, self).__init__(ps)
self._should_filter_suites = should_filter_suites
self._filtered_suite_names = filtered_suite_names
self._enable_smoke_test_mode = enable_smoke_test_mode
self._enabled_suites = []
@staticmethod
def GetFullSuiteName(name):
return _SPEEDOMETER_SUITE_NAME_BASE.format(name)
@staticmethod
def GetSuites(suite_regex):
if not suite_regex:
return []
exp = re.compile(suite_regex)
return [name for name in _SPEEDOMETER_SUITES
if exp.search(Speedometer2Story.GetFullSuiteName(name))]
def ExecuteTest(self, action_runner):
action_runner.tab.WaitForDocumentReadyStateToBeComplete()
iterationCount = 10
# A single iteration on android takes ~75 seconds, the benchmark times out
# when running for 10 iterations.
if action_runner.tab.browser.platform.GetOSName() == 'android':
iterationCount = 3
# For a smoke test one iteration is sufficient
if self._enable_smoke_test_mode:
iterationCount = 1
if self._should_filter_suites:
action_runner.ExecuteJavaScript("""
Suites.forEach(function(suite) {
suite.disabled = {{ filtered_suites }}.indexOf(suite.name) < 0;
});
""", filtered_suites=self._filtered_suites)
self._enabled_suites = action_runner.EvaluateJavaScript("""
(function() {
var suitesNames = [];
Suites.forEach(function(s) {
if (!s.disabled)
suitesNames.push(s.name);
});
return suitesNames;
})();""")
action_runner.ExecuteJavaScript("""
// Store all the results in the benchmarkClient
var testDone = false;
var iterationCount = {{ count }};
var benchmarkClient = {};
var suiteValues = [];
benchmarkClient.didRunSuites = function(measuredValues) {
suiteValues.push(measuredValues);
};
benchmarkClient.didFinishLastIteration = function () {
testDone = true;
};
var runner = new BenchmarkRunner(Suites, benchmarkClient);
runner.runMultipleIterations(iterationCount);
""",
count=iterationCount)
action_runner.WaitForJavaScriptCondition('testDone', timeout=600)
def ParseTestResults(self, action_runner):
if not self._should_filter_suites:
self.AddJavascriptMetricValue(list_of_scalar_values.ListOfScalarValues(
self, 'Total', 'ms',
action_runner.EvaluateJavaScript(
'suiteValues.map(each => each.total)'),
important=True))
self.AddJavascriptMetricValue(list_of_scalar_values.ListOfScalarValues(
self, 'RunsPerMinute', 'score',
action_runner.EvaluateJavaScript(
'suiteValues.map(each => each.score)'),
important=True))
# Extract the timings for each suite
for suite_name in self._enabled_suites:
self.AddJavascriptMetricValue(list_of_scalar_values.ListOfScalarValues(
self, suite_name, 'ms',
action_runner.EvaluateJavaScript("""
var suite_times = [];
for(var i = 0; i < iterationCount; i++) {
suite_times.push(
suiteValues[i].tests[{{ key }}].total);
};
suite_times;
""",
key=suite_name), important=False))
tools/perf/page_sets/speedometer_pages.py 0 → 100644
View file @ a3dd9802
# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry import story
from telemetry.value import list_of_scalar_values
from page_sets import press_story
class SpeedometerStory(press_story.PressStory):
URL='http://browserbench.org/Speedometer/'
enabled_suites = [
'VanillaJS-TodoMVC',
'EmberJS-TodoMVC',
'BackboneJS-TodoMVC',
'jQuery-TodoMVC',
'AngularJS-TodoMVC',
'React-TodoMVC',
'FlightJS-TodoMVC'
]
def ExecuteTest(self, action_runner):
action_runner.tab.WaitForDocumentReadyStateToBeComplete()
iterationCount = 10
# A single iteration on android takes ~75 seconds, the benchmark times out
# when running for 10 iterations.
if action_runner.tab.browser.platform.GetOSName() == 'android':
iterationCount = 3
action_runner.ExecuteJavaScript("""
// Store all the results in the benchmarkClient
benchmarkClient._measuredValues = []
benchmarkClient.didRunSuites = function(measuredValues) {
benchmarkClient._measuredValues.push(measuredValues);
benchmarkClient._timeValues.push(measuredValues.total);
};
benchmarkClient.iterationCount = {{ count }};
startTest();
""",
count=iterationCount)
action_runner.WaitForJavaScriptCondition(
'benchmarkClient._finishedTestCount == benchmarkClient.testsCount',
timeout=600)
def ParseTestResults(self, action_runner):
self.AddJavascriptMetricValue(list_of_scalar_values.ListOfScalarValues(
self, 'Total', 'ms',
action_runner.EvaluateJavaScript('benchmarkClient._timeValues'),
important=True))
self.AddJavascriptMetricValue(list_of_scalar_values.ListOfScalarValues(
self, 'RunsPerMinute', 'score',
action_runner.EvaluateJavaScript(
'[parseFloat(document.getElementById("result-number").innerText)];'
),
important=True))
# Extract the timings for each suite
for suite_name in self.enabled_suites:
self.AddJavascriptMetricValue(list_of_scalar_values.ListOfScalarValues(
self, suite_name, 'ms',
action_runner.EvaluateJavaScript("""
var suite_times = [];
for(var i = 0; i < benchmarkClient.iterationCount; i++) {
suite_times.push(
benchmarkClient._measuredValues[i].tests[{{ key }}].total);
};
suite_times;
""",
key=suite_name), important=False))
class SpeedometerStorySet(story.StorySet):
def __init__(self):
super(SpeedometerStorySet, self).__init__(
archive_data_file='data/speedometer.json',
cloud_storage_bucket=story.PUBLIC_BUCKET)
self.AddStory(SpeedometerStory(self))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment