Commit 83dc32e0 authored by nednguyen's avatar nednguyen Committed by Commit bot

Kill endure benchmark and all that related to it.

BUG=

Review URL: https://codereview.chromium.org/616363002

Cr-Commit-Position: refs/heads/master@{#299008}
parent 53aebab8
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""The Endure benchmarks measure memory performance over a period of time.
In each Endure benchmark, one page action is performed repeatedly and memory
usage is measured periodically. The specific page actions are defined in the
page sets, and the statistics that are gathered are determined by the Endure
measurement class.
"""
from measurements import endure
import page_sets
from telemetry import benchmark
class _EndureBenchmark(benchmark.Benchmark):
"""Base class which sets options for endure benchmarks below."""
test = endure.Endure
# Default options for endure benchmarks. Could be overridden in subclasses.
options = {
# Depending on the page and the actions performed on the page,
# 1000 iterations should be between 30 and 60 minutes.
'page_repeat': 1000,
# One sample per 10 iterations -> 200 points per run.
'perf_stats_interval': 10
}
@benchmark.Disabled
class EndureCalendarForwardBackward(_EndureBenchmark):
page_set = page_sets.CalendarForwardBackwardPageSet
@benchmark.Disabled
class EndureBrowserControl(_EndureBenchmark):
page_set = page_sets.BrowserControlPageSet
@benchmark.Disabled
class EndureBrowserControlClick(_EndureBenchmark):
page_set = page_sets.BrowserControlClickPageSet
@benchmark.Disabled
class EndureGmailAltThreadlistConversation(_EndureBenchmark):
page_set = page_sets.GmailAltThreadlistConversationPageSet
@benchmark.Disabled
class EndureGmailAltTwoLabels(_EndureBenchmark):
page_set = page_sets.GmailAltTwoLabelsPageSet
@benchmark.Disabled
class EndureGmailExpandCollapseConversation(_EndureBenchmark):
page_set = page_sets.GmailExpandCollapseConversationPageSet
@benchmark.Disabled
class EndureIndexedDBOffline(_EndureBenchmark):
page_set = page_sets.IndexeddbOfflinePageSet
@benchmark.Disabled
class EndurePlusAltPostsPhotos(_EndureBenchmark):
page_set = page_sets.PlusAltPostsPhotosPageSet
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import optparse
import time
from metrics import v8_object_stats
from telemetry.page import page_test
from telemetry.value import scalar
# V8 statistics counter names. These can be retrieved using
# v8_object_stats.V8ObjectStatsMetric.GetV8StatsTable.
_V8_BYTES_COMMITTED = [
'V8.MemoryNewSpaceBytesCommitted',
'V8.MemoryOldPointerSpaceBytesCommitted',
'V8.MemoryOldDataSpaceBytesCommitted',
'V8.MemoryCodeSpaceBytesCommitted',
'V8.MemoryMapSpaceBytesCommitted',
'V8.MemoryCellSpaceBytesCommitted',
'V8.MemoryPropertyCellSpaceBytesCommitted',
'V8.MemoryLoSpaceBytesCommitted',
]
_V8_BYTES_USED = [
'V8.MemoryNewSpaceBytesUsed',
'V8.MemoryOldPointerSpaceBytesUsed',
'V8.MemoryOldDataSpaceBytesUsed',
'V8.MemoryCodeSpaceBytesUsed',
'V8.MemoryMapSpaceBytesUsed',
'V8.MemoryCellSpaceBytesUsed',
'V8.MemoryPropertyCellSpaceBytesUsed',
'V8.MemoryLoSpaceBytesUsed',
]
_V8_MEMORY_ALLOCATED = [
'V8.OsMemoryAllocated',
]
# NOTE(chrishenry): This measurement does NOT work anymore. The
# feature it depends on has been removed from telemetry. The benchmark
# has been disabled on bot.
class Endure(page_test.PageTest):
def __init__(self):
super(Endure, self).__init__('RunEndure')
# Browser object, saved so that browser.memory_stats can be accessed.
self._browser = None
# Dictionary of trace name to lists of y-values, for making summary values.
self._y_values = {}
# Number of page repetitions since the start of the test.
self._iterations_elapsed = 0
# Start time of the test, used to report total time.
self._start_time = None
@classmethod
def AddCommandLineArgs(cls, parser):
group = optparse.OptionGroup(parser, 'Endure options')
group.add_option('--perf-stats-interval',
dest='perf_stats_interval',
default=1,
type='int',
help='Number of iterations per sampling of statistics.')
parser.add_option_group(group)
def DidStartBrowser(self, browser):
"""Initializes the measurement after the browser is started."""
self._browser = browser
self._start_time = time.time()
def CustomizeBrowserOptions(self, options):
"""Adds extra command-line options to the browser."""
v8_object_stats.V8ObjectStatsMetric.CustomizeBrowserOptions(options)
def ValidateAndMeasurePage(self, page, tab, results):
"""Takes a sample and adds a result if enough time has passed."""
self._iterations_elapsed += 1
if self._iterations_elapsed % int(self.options.perf_stats_interval) == 0:
self._SampleStats(tab, results)
def _SampleStats(self, tab, results):
"""Records information and add it to the results."""
def AddPoint(trace_name, units_y, value_y, chart_name=None):
"""Adds one data point to the results object."""
if chart_name:
trace_name = '%s.%s' % (chart_name, trace_name)
else:
assert '.' not in trace_name, (
'Trace names cannot contain "." with an empty chart_name since this'
' is used to delimit chart_name.trace_name.')
results.AddValue(scalar.ScalarValue(
results.current_page, trace_name + '_X', 'iterations',
self._iterations_elapsed, important=False))
results.AddValue(scalar.ScalarValue(
results.current_page, trace_name + '_Y', units_y, value_y,
important=False))
# Save the value so that summary stats can be calculated.
if trace_name not in self._y_values:
self._y_values[trace_name] = {
'units': units_y,
'chart_name': chart_name,
'values': [],
}
self._y_values[trace_name]['values'].append(value_y)
# DOM nodes and event listeners.
dom_stats = tab.dom_stats
dom_node_count = dom_stats['node_count']
event_listener_count = dom_stats['event_listener_count']
AddPoint('dom_nodes', 'count', dom_node_count, chart_name='object_counts')
AddPoint('event_listeners', 'count', event_listener_count,
chart_name='object_counts')
# Browser and renderer virtual memory stats.
memory_stats = self._browser.memory_stats
def BrowserVMStats(statistic_name):
"""Get VM stats from the Browser object in KB."""
return memory_stats[statistic_name].get('VM', 0) / 1024.0
AddPoint('browser_vm', 'KB', BrowserVMStats('Browser'),
chart_name='vm_stats')
AddPoint('renderer_vm', 'KB', BrowserVMStats('Renderer'),
chart_name='vm_stats')
AddPoint('gpu_vm', 'KB', BrowserVMStats('Gpu'), chart_name='vm_stats')
# V8 counter stats.
def V8StatsSum(counters):
"""Given a list of V8 counter names, get the sum of the values in KB."""
stats = v8_object_stats.V8ObjectStatsMetric.GetV8StatsTable(tab, counters)
return sum(stats.values()) / 1024.0
AddPoint('v8_memory_committed', 'KB', V8StatsSum(_V8_BYTES_COMMITTED),
chart_name='v8_counter_stats')
AddPoint('v8_memory_used', 'KB', V8StatsSum(_V8_BYTES_USED),
chart_name='v8_counter_stats')
AddPoint('v8_memory_allocated', 'KB', V8StatsSum(_V8_MEMORY_ALLOCATED),
chart_name='v8_counter_stats')
def DidRunTest(self, browser, results):
"""Adds summary results (single number for one test run)."""
# Report test run length.
results.AddSummaryValue(scalar.ScalarValue(None, 'total_iterations',
'iterations',
self._iterations_elapsed,
important=False))
results.AddSummaryValue(scalar.ScalarValue(None, 'total_time', 'seconds',
time.time() - self._start_time,
important=False))
# Add summary stats which could be monitored for anomalies.
for trace_name in self._y_values:
units = self._y_values[trace_name]['units']
chart_name = self._y_values[trace_name]['chart_name']
values = self._y_values[trace_name]['values']
value_name = '%s.%s_max' % (chart_name, trace_name)
results.AddSummaryValue(
scalar.ScalarValue(None, value_name, units, max(values)))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
class BrowserControlPage(page_module.Page):
""" Why: Continually attach and detach a DOM tree from a basic document. """
def __init__(self, page_set):
super(BrowserControlPage, self).__init__(
url='file://endure/browser_control.html',
page_set=page_set,
name='browser_control')
self.user_agent_type = 'desktop'
def RunEndure(self, action_runner):
action_runner.Wait(2)
class BrowserControlPageSet(page_set_module.PageSet):
""" Chrome Endure control test for the browser. """
def __init__(self):
super(BrowserControlPageSet, self).__init__(
user_agent_type='desktop')
self.AddPage(BrowserControlPage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
class BrowserControlClickPage(page_module.Page):
""" Why: Use a JavaScript .click() call to attach and detach a DOM tree
from a basic document.
"""
def __init__(self, page_set):
super(BrowserControlClickPage, self).__init__(
url='file://endure/browser_control_click.html',
page_set=page_set,
name='browser_control_click')
self.user_agent_type = 'desktop'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.WaitForElement('#attach')
def RunEndure(self, action_runner):
action_runner.ClickElement('#attach')
action_runner.Wait(0.5)
action_runner.ClickElement('#detach')
action_runner.Wait(0.5)
class BrowserControlClickPageSet(page_set_module.PageSet):
""" Chrome Endure control test for the browser. """
def __init__(self):
super(BrowserControlClickPageSet, self).__init__(
user_agent_type='desktop')
self.AddPage(BrowserControlClickPage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
class CalendarForwardBackwardPage(page_module.Page):
""" Why: Click forward(4x) and backwards(4x) repeatedly """
def __init__(self, page_set):
super(CalendarForwardBackwardPage, self).__init__(
url='https://www.google.com/calendar/',
page_set=page_set,
name='calendar_forward_backward',
credentials_path = 'data/credentials.json')
self.credentials = 'google'
self.user_agent_type = 'desktop'
self.archive_data_file = 'data/calendar_forward_backward.json'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navForward"]')
action_runner.ExecuteJavaScript('''
(function() {
var elem = document.createElement('meta');
elem.name='viewport';
elem.content='initial-scale=1';
document.body.appendChild(elem);
})();''')
def RunEndure(self, action_runner):
action_runner.ClickElement('div[class~="navForward"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navForward"]')
action_runner.ClickElement('div[class~="navForward"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navForward"]')
action_runner.ClickElement('div[class~="navForward"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navForward"]')
action_runner.ClickElement('div[class~="navForward"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navBack"]')
action_runner.ClickElement('div[class~="navBack"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navBack"]')
action_runner.ClickElement('div[class~="navBack"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navBack"]')
action_runner.ClickElement('div[class~="navBack"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navBack"]')
action_runner.ClickElement('div[class~="navBack"]')
action_runner.Wait(2)
action_runner.WaitForElement('div[class~="navForward"]')
class CalendarForwardBackwardPageSet(page_set_module.PageSet):
""" Chrome Endure test for Google Calendar. """
def __init__(self):
super(CalendarForwardBackwardPageSet, self).__init__(
user_agent_type='desktop',
archive_data_file='data/calendar_forward_backward.json',
bucket=page_set_module.PUBLIC_BUCKET)
self.AddPage(CalendarForwardBackwardPage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import re
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
def _CreateXpathFunction(xpath):
return ('document.evaluate("%s",'
'document,'
'null,'
'XPathResult.FIRST_ORDERED_NODE_TYPE,'
'null)'
'.singleNodeValue' % re.escape(xpath))
def _GetCurrentLocation(action_runner):
return action_runner.EvaluateJavaScript('document.location.href')
def _WaitForLocationChange(action_runner, old_href):
action_runner.WaitForJavaScriptCondition(
'document.location.href != "%s"' % old_href)
class GmailAltThreadlistConversationPage(
page_module.Page):
""" Why: Alternate between Inbox and the first email conversation. """
def __init__(self, page_set):
super(GmailAltThreadlistConversationPage, self).__init__(
url='https://mail.google.com/mail/',
page_set=page_set,
name='gmail_alt_threadlist_conversation',
credentials_path = 'data/credentials.json')
self.user_agent_type = 'desktop'
self.archive_data_file = 'data/gmail_alt_threadlist_conversation.json'
self.credentials = 'google'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.WaitForJavaScriptCondition(
'window.gmonkey !== undefined && '
'document.getElementById("gb") !== null')
def RunEndure(self, action_runner):
old_href = _GetCurrentLocation(action_runner)
action_runner.ClickElement(
element_function=_CreateXpathFunction('//span[@email]'))
_WaitForLocationChange(action_runner, old_href)
action_runner.Wait(1)
old_href = _GetCurrentLocation(action_runner)
action_runner.ClickElement(
'a[href="https://mail.google.com/mail/u/0/?shva=1#inbox"]')
_WaitForLocationChange(action_runner, old_href)
action_runner.Wait(1)
class GmailAltThreadlistConversationPageSet(page_set_module.PageSet):
""" Chrome Endure test for GMail. """
def __init__(self):
super(GmailAltThreadlistConversationPageSet, self).__init__(
user_agent_type='desktop',
archive_data_file='data/gmail_alt_threadlist_conversation.json',
bucket=page_set_module.PUBLIC_BUCKET)
self.AddPage(GmailAltThreadlistConversationPage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
def _GetCurrentLocation(action_runner):
return action_runner.EvaluateJavaScript('document.location.href')
def _WaitForLocationChange(action_runner, old_href):
action_runner.WaitForJavaScriptCondition(
'document.location.href != "%s"' % old_href)
class GmailAltTwoLabelsPage(page_module.Page):
""" Why: Alternate between Inbox and Sent Mail """
def __init__(self, page_set):
super(GmailAltTwoLabelsPage, self).__init__(
url='https://mail.google.com/mail/',
page_set=page_set,
name='gmail_alt_two_labels',
credentials_path = 'data/credentials.json')
self.credentials = 'google'
self.user_agent_type = 'desktop'
self.archive_data_file = 'data/gmail_alt_two_labels.json'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.WaitForJavaScriptCondition(
'window.gmonkey !== undefined && '
'document.getElementById("gb") !== null')
def RunEndure(self, action_runner):
old_href = _GetCurrentLocation(action_runner)
action_runner.ClickElement(
'a[href="https://mail.google.com/mail/u/0/?shva=1#sent"]')
_WaitForLocationChange(action_runner, old_href)
action_runner.Wait(1)
old_href = _GetCurrentLocation(action_runner)
action_runner.ClickElement(
'a[href="https://mail.google.com/mail/u/0/?shva=1#inbox"]')
_WaitForLocationChange(action_runner, old_href)
action_runner.Wait(1)
class GmailAltTwoLabelsPageSet(page_set_module.PageSet):
""" Chrome Endure test for GMail. """
def __init__(self):
super(GmailAltTwoLabelsPageSet, self).__init__(
user_agent_type='desktop',
archive_data_file='data/gmail_alt_two_labels.json',
bucket=page_set_module.PUBLIC_BUCKET)
self.AddPage(GmailAltTwoLabelsPage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
class GmailExpandCollapseConversationPage(
page_module.Page):
""" Why: Expand and Collapse a long conversation. """
# TODO(edmundyan): Find a long conversation rather than hardcode url
def __init__(self, page_set):
super(GmailExpandCollapseConversationPage, self).__init__(
url='https://mail.google.com/mail/u/0/#inbox/13c6a141fa95ffe0',
page_set=page_set,
name='gmail_expand_collapse_conversation',
credentials_path='data/credentials.json')
self.credentials = 'google'
self.user_agent_type = 'desktop'
self.archive_data_file = 'data/gmail_expand_collapse_conversation.json'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.WaitForElement('img[alt="Expand all"]')
action_runner.ClickElement('img[alt="Expand all"]')
action_runner.Wait(5)
action_runner.WaitForElement('img[alt="Collapse all"]')
action_runner.ClickElement('img[alt="Collapse all"]')
action_runner.Wait(1)
def RunEndure(self, action_runner):
action_runner.WaitForElement('img[alt="Expand all"]')
action_runner.ClickElement('img[alt="Expand all"]')
action_runner.Wait(1)
action_runner.WaitForElement('img[alt="Collapse all"]')
action_runner.ClickElement('img[alt="Collapse all"]')
action_runner.Wait(1)
class GmailExpandCollapseConversationPageSet(page_set_module.PageSet):
"""
Description: Chrome Endure test for GMail.
"""
def __init__(self):
super(GmailExpandCollapseConversationPageSet, self).__init__(
user_agent_type='desktop',
archive_data_file='data/gmail_expand_collapse_conversation.json',
bucket=page_set_module.PUBLIC_BUCKET)
self.AddPage(GmailExpandCollapseConversationPage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import re
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
def _CreateXpathFunction(xpath):
return ('document.evaluate("%s",'
'document,'
'null,'
'XPathResult.FIRST_ORDERED_NODE_TYPE,'
'null)'
'.singleNodeValue' % re.escape(xpath))
class IndexeddbOfflinePage(page_module.Page):
""" Why: Simulates user input while offline and sync while online. """
def __init__(self, page_set):
super(IndexeddbOfflinePage, self).__init__(
url='file://endure/indexeddb_app.html',
page_set=page_set,
name='indexeddb_offline')
self.user_agent_type = 'desktop'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.WaitForElement(text='initialized')
def RunEndure(self, action_runner):
action_runner.WaitForElement('button[id="online"]:not(disabled)')
action_runner.ClickElement('button[id="online"]:not(disabled)')
action_runner.WaitForElement(
element_function=_CreateXpathFunction('id("state")[text()="online"]'))
action_runner.Wait(1)
action_runner.WaitForElement('button[id="offline"]:not(disabled)')
action_runner.ClickElement('button[id="offline"]:not(disabled)')
action_runner.WaitForElement(
element_function=_CreateXpathFunction('id("state")[text()="offline"]'))
class IndexeddbOfflinePageSet(page_set_module.PageSet):
""" Chrome Endure test for IndexedDB. """
def __init__(self):
super(IndexeddbOfflinePageSet, self).__init__(
user_agent_type='desktop')
self.AddPage(IndexeddbOfflinePage(self))
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from telemetry.page import page as page_module
from telemetry.page import page_set as page_set_module
class PlusAltPostsPhotosPage(page_module.Page):
""" Why: Alternate between clicking posts and albums """
def __init__(self, page_set):
super(PlusAltPostsPhotosPage, self).__init__(
url='https://plus.google.com/+BarackObama/posts',
page_set=page_set,
name='plus_alt_posts_photos',
credentials_path = 'data/credentials.json')
self.credentials = 'google'
self.user_agent_type = 'desktop'
self.archive_data_file = 'data/plus_alt_posts_photos.json'
def RunNavigateSteps(self, action_runner):
action_runner.NavigateToPage(self)
action_runner.WaitForElement(text='Barack Obama')
action_runner.WaitForElement(
'span[guidedhelpid="posts_tab_profile"][class*="s6U8x"]')
def RunEndure(self, action_runner):
action_runner.ClickElement('span[guidedhelpid="posts_tab_profile"]')
action_runner.WaitForElement(
'span[guidedhelpid="posts_tab_profile"][class*="s6U8x"]')
action_runner.Wait(5)
action_runner.ClickElement('span[guidedhelpid="photos_tab_profile"]')
action_runner.WaitForElement(
'span[guidedhelpid="photos_tab_profile"][class*="s6U8x"]')
action_runner.Wait(5)
class PlusAltPostsPhotosPageSet(page_set_module.PageSet):
""" Chrome Endure test for Google Plus. """
def __init__(self):
super(PlusAltPostsPhotosPageSet, self).__init__(
user_agent_type='desktop',
archive_data_file='data/plus_alt_posts_photos.json',
bucket=page_set_module.PUBLIC_BUCKET)
self.AddPage(PlusAltPostsPhotosPage(self))
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment