Skip to content

GitLab

T
tangled
Commit 5136cc20 authored by rnephew's avatar rnephew Committed by Commit Bot
Browse files
Options

[Telemetry] Move ShouldDisable to StoryExpectations where possible. 

There are some ShouldDisables that cannot switch over, such as
BattOr detecting and reference build detecting.

Bug: 713222
Change-Id: Ie6ce17c6f41d7df3c480b47d7560246f4b129ed7
Reviewed-on: https://chromium-review.googlesource.com/661943
Commit-Queue: rnephew <rnephew@chromium.org>
Reviewed-by: default avatarNed Nguyen <nednguyen@google.com>
Reviewed-by: default avatarStephen Martinis <martiniss@chromium.org>
Cr-Commit-Position: refs/heads/master@{#501443}
parent e9713155
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 301 additions and 1805 deletions
testing/buildbot/chromium.perf.fyi.json
View file @ 5136cc20
......@@ -1408,38 +1408,6 @@
"upload_test_results": false
}
},
{
"args": [
"loading.desktop",
"-v",
"--upload-results",
"--browser=reference",
"--output-format=chartjson",
"--max-failures=5",
"--output-trace-tag=_ref"
],
"isolate_name": "telemetry_perf_tests",
"name": "loading.desktop.reference",
"override_compile_targets": [
"telemetry_perf_tests"
],
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"gpu": "8086:22b1",
"id": "build137-b1",
"os": "Windows-10-10586",
"pool": "Chrome-perf-fyi"
}
],
"expiration": 36000,
"hard_timeout": 14400,
"ignore_task_failure": true,
"io_timeout": 600,
"upload_test_results": false
}
},
{
"args": [
"media.media_cns_cases",
......@@ -4456,38 +4424,6 @@
"upload_test_results": false
}
},
{
"args": [
"v8.runtime_stats.top_25",
"-v",
"--upload-results",
"--browser=reference",
"--output-format=chartjson",
"--max-failures=5",
"--output-trace-tag=_ref"
],
"isolate_name": "telemetry_perf_tests",
"name": "v8.runtime_stats.top_25.reference",
"override_compile_targets": [
"telemetry_perf_tests"
],
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"gpu": "8086:22b1",
"id": "build147-b1",
"os": "Windows-10-10586",
"pool": "Chrome-perf-fyi"
}
],
"expiration": 36000,
"hard_timeout": 10800,
"ignore_task_failure": true,
"io_timeout": 600,
"upload_test_results": false
}
},
{
"args": [
"v8.runtimestats.browsing_desktop",
......@@ -5886,38 +5822,6 @@
"upload_test_results": false
}
},
{
"args": [
"loading.desktop",
"-v",
"--upload-results",
"--browser=reference",
"--output-format=chartjson",
"--max-failures=5",
"--output-trace-tag=_ref"
],
"isolate_name": "telemetry_perf_tests",
"name": "loading.desktop.reference",
"override_compile_targets": [
"telemetry_perf_tests"
],
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"gpu": "1002:9874",
"id": "build186-b4",
"os": "Windows-10-10586",
"pool": "Chrome-perf-fyi"
}
],
"expiration": 36000,
"hard_timeout": 14400,
"ignore_task_failure": true,
"io_timeout": 600,
"upload_test_results": false
}
},
{
"args": [
"media.media_cns_cases",
......@@ -8892,38 +8796,6 @@
"upload_test_results": false
}
},
{
"args": [
"v8.runtime_stats.top_25",
"-v",
"--upload-results",
"--browser=reference",
"--output-format=chartjson",
"--max-failures=5",
"--output-trace-tag=_ref"
],
"isolate_name": "telemetry_perf_tests",
"name": "v8.runtime_stats.top_25.reference",
"override_compile_targets": [
"telemetry_perf_tests"
],
"swarming": {
"can_use_on_swarming_builders": true,
"dimension_sets": [
{
"gpu": "1002:9874",
"id": "build211-b4",
"os": "Windows-10-10586",
"pool": "Chrome-perf-fyi"
}
],
"expiration": 36000,
"hard_timeout": 10800,
"ignore_task_failure": true,
"io_timeout": 600,
"upload_test_results": false
}
},
{
"args": [
"v8.runtimestats.browsing_desktop",
......
testing/buildbot/chromium.perf.json
View file @ 5136cc20
This diff is collapsed.
tools/perf/benchmarks/loading.py
View file @ 5136cc20
......@@ -29,10 +29,6 @@ class LoadingDesktop(_LoadingBase):
""" A benchmark measuring loading performance of desktop sites. """
SUPPORTED_PLATFORMS = [story.expectations.ALL_DESKTOP]
@classmethod
def ShouldDisable(cls, possible_browser):
return possible_browser.browser_type == 'reference'
def CreateStorySet(self, options):
return page_sets.LoadingDesktopStorySet(
cache_temperatures=[cache_temperature.PCV1_COLD,
......@@ -57,20 +53,6 @@ class LoadingMobile(_LoadingBase):
""" A benchmark measuring loading performance of mobile sites. """
SUPPORTED_PLATFORMS = [story.expectations.ALL_MOBILE]
@classmethod
def ShouldDisable(cls, possible_browser):
# crbug.com/619254
if possible_browser.browser_type == 'reference':
return True
# crbug.com/676612
if ((possible_browser.platform.GetDeviceTypeName() == 'Nexus 6' or
possible_browser.platform.GetDeviceTypeName() == 'AOSP on Shamu') and
possible_browser.browser_type == 'android-webview'):
return True
return False
def CreateStorySet(self, options):
return page_sets.LoadingMobileStorySet(
cache_temperatures=[cache_temperature.ANY],
......@@ -79,6 +61,8 @@ class LoadingMobile(_LoadingBase):
def GetExpectations(self):
class StoryExpectations(story.expectations.StoryExpectations):
def SetExpectations(self):
self.DisableBenchmark(
[story.expectations.ANDROID_NEXUS6_WEBVIEW], 'crbug.com/676612')
self.DisableStory('GFK', [story.expectations.ALL],
'N5X Timeout issue: crbug.com/702175')
self.DisableStory('MLSMatrix', [story.expectations.ALL],
......
tools/perf/benchmarks/memory.py
View file @ 5136cc20
......@@ -180,17 +180,12 @@ class MemoryLongRunningIdleGmail(_MemoryV8Benchmark):
def Name(cls):
return 'memory.long_running_idle_gmail_tbmv2'
@classmethod
def ShouldDisable(cls, possible_browser):
return (cls.IsSvelte(possible_browser) or # http://crbug.com/611167
# http://crbug.com/671650
((possible_browser.browser_type == 'reference' and
possible_browser.platform.GetDeviceTypeName() == 'Nexus 5')))
def GetExpectations(self):
class StoryExpectations(story.expectations.StoryExpectations):
def SetExpectations(self):
pass # Nothing disabled.
self.DisableBenchmark(
[story.expectations.ANDROID_SVELTE],
'Requires a lot of memory: crbug.com/611167')
return StoryExpectations()
......@@ -208,12 +203,10 @@ class MemoryLongRunningIdleGmailBackground(_MemoryV8Benchmark):
def Name(cls):
return 'memory.long_running_idle_gmail_background_tbmv2'
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/616530
return cls.IsSvelte(possible_browser)
def GetExpectations(self):
class StoryExpectations(story.expectations.StoryExpectations):
def SetExpectations(self):
pass
self.DisableBenchmark(
[story.expectations.ANDROID_SVELTE],
'Requires a lot of memory: crbug.com/616530')
return StoryExpectations()
tools/perf/benchmarks/oortonline.py
View file @ 5136cc20
......@@ -107,10 +107,6 @@ class OortOnlineTBMv2(perf_benchmark.PerfBenchmark):
chrome_trace_config.MemoryDumpConfig())
return options
@classmethod
def ShouldDisable(cls, possible_browser):
return possible_browser.platform.GetDeviceTypeName() == 'Nexus 9'
@classmethod
def Name(cls):
return 'oortonline_tbmv2'
......
tools/perf/benchmarks/smoothness.py
View file @ 5136cc20
# Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import multiprocessing
from core import perf_benchmark
from benchmarks import silk_flags
......@@ -50,18 +48,6 @@ class SmoothnessTop25(_Smoothness):
def Name(cls):
return 'smoothness.top_25_smooth'
@classmethod
def ShouldDisable(cls, possible_browser):
# http://crbug.com/597656
if (possible_browser.browser_type == 'reference' and
possible_browser.platform.GetDeviceTypeName() == 'Nexus 5X'):
return True
# http://crbug.com/650762
if (possible_browser.browser_type == 'reference' and
possible_browser.platform.GetOSName() == 'win'):
return True
return False
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
......@@ -88,21 +74,11 @@ class SmoothnessToughFiltersCases(_Smoothness):
def Name(cls):
return 'smoothness.tough_filters_cases'
@classmethod
def ShouldDisable(cls, possible_browser):
# http://crbug.com/616520
if (cls.IsSvelte(possible_browser) and
possible_browser.browser_type == 'reference'):
return True
# http://crbug.com/624032
if possible_browser.platform.GetDeviceTypeName() == 'Nexus 6':
return True
return False
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
pass # Nothing.
self.DisableBenchmark([story_module.expectations.ANDROID_NEXUS6],
'crbug.com/624032')
return StoryExpectations()
......@@ -222,11 +198,6 @@ class SmoothnessKeyMobileSites(_Smoothness):
def Name(cls):
return 'smoothness.key_mobile_sites_smooth'
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/597656
return (possible_browser.browser_type == 'reference' and
possible_browser.platform.GetDeviceTypeName() == 'Nexus 5X')
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
......@@ -245,12 +216,6 @@ class SmoothnessToughAnimationCases(_Smoothness):
def Name(cls):
return 'smoothness.tough_animation_cases'
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/595737
# This test is flaky on low-end windows machine.
return (possible_browser.platform.GetOSName() == 'win' and
multiprocessing.cpu_count() <= 2)
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
......@@ -320,11 +285,6 @@ class SmoothnessGpuRasterizationTop25(_Smoothness):
def Name(cls):
return 'smoothness.gpu_rasterization.top_25_smooth'
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/597656
return (possible_browser.browser_type == 'reference' and
possible_browser.platform.GetDeviceTypeName() == 'Nexus 5X')
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
......@@ -386,11 +346,6 @@ class SmoothnessGpuRasterizationFiltersCases(_Smoothness):
def Name(cls):
return 'smoothness.gpu_rasterization.tough_filters_cases'
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/616540
return (cls.IsSvelte(possible_browser) and
possible_browser.browser_type == 'reference')
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
......@@ -414,11 +369,6 @@ class SmoothnessSyncScrollKeyMobileSites(_Smoothness):
def Name(cls):
return 'smoothness.sync_scroll.key_mobile_sites_smooth'
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/597656
return (possible_browser.browser_type == 'reference' and
possible_browser.platform.GetDeviceTypeName() == 'Nexus 5X')
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
......@@ -756,17 +706,11 @@ class SmoothnessPathologicalMobileSites(_Smoothness):
def Name(cls):
return 'smoothness.pathological_mobile_sites'
@classmethod
def ShouldDisable(cls, possible_browser):
# http://crbug.com/685342
if possible_browser.platform.GetDeviceTypeName() == 'Nexus 7':
return True
return False
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
pass
self.DisableBenchmark([story_module.expectations.ANDROID_NEXUS7],
'crbug.com/685342')
return StoryExpectations()
......@@ -793,10 +737,6 @@ class SmoothnessToughAdCases(_Smoothness):
def Name(cls):
return 'smoothness.tough_ad_cases'
@classmethod
def ShouldDisable(cls, possible_browser):
return cls.IsSvelte(possible_browser) # http://crbug.com/555089
@classmethod
def ValueCanBeAddedPredicate(cls, value, is_first_result):
del is_first_result # unused
......@@ -806,7 +746,8 @@ class SmoothnessToughAdCases(_Smoothness):
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
pass # Nothing.
self.DisableBenchmark([story_module.expectations.ANDROID_SVELTE],
'www.crbug.com/555089')
return StoryExpectations()
......@@ -819,14 +760,11 @@ class SmoothnessToughWebGLAdCases(_Smoothness):
def Name(cls):
return 'smoothness.tough_webgl_ad_cases'
@classmethod
def ShouldDisable(cls, possible_browser):
return cls.IsSvelte(possible_browser) # http://crbug.com/574485
def GetExpectations(self):
class StoryExpectations(story_module.expectations.StoryExpectations):
def SetExpectations(self):
pass # Nothing.
self.DisableBenchmark([story_module.expectations.ANDROID_SVELTE],
'crbug.com/574485')
return StoryExpectations()
......
tools/perf/benchmarks/start_with_url.py
View file @ 5136cc20
......@@ -43,11 +43,6 @@ class StartWithUrlColdTBM(_StartupPerfBenchmark):
options.clear_sytem_cache_for_browser_and_profile_on_start = True
super(StartWithUrlColdTBM, self).SetExtraBrowserOptions(options)
@classmethod
def ShouldDisable(cls, possible_browser): # http://crbug.com/667470
return (possible_browser.platform.GetDeviceTypeName() in
['Nexus 7v2', 'Nexus 9'])
@classmethod
def Name(cls):
return 'start_with_url.cold.startup_pages'
......@@ -56,6 +51,8 @@ class StartWithUrlColdTBM(_StartupPerfBenchmark):
def GetExpectations(self):
class StoryExpectations(story.expectations.StoryExpectations):
def SetExpectations(self):
self.DisableBenchmark(
[story.expectations.ANDROID_NEXUS7], 'crbug.com/667470')
self.DisableStory(
'http://kapook.com', [story.expectations.ALL], 'crbug.com/667470')
return StoryExpectations()
......
tools/perf/benchmarks/v8.py
View file @ 5136cc20
......@@ -25,13 +25,6 @@ class V8DetachedContextAgeInGC(perf_benchmark.PerfBenchmark):
def Name(cls):
return 'v8.detached_context_age_in_gc'
@classmethod
def ShouldDisable(cls, possible_browser):
# http://crbug.com/685350
if possible_browser.platform.GetDeviceTypeName() == 'Nexus 9':
return True
return False
def GetExpectations(self):
class StoryExpectations(story.expectations.StoryExpectations):
def SetExpectations(self):
......
tools/perf/core/perf_data_generator.py
View file @ 5136cc20
......@@ -830,7 +830,10 @@ BLACKLISTED_DEVICES = []
# List of benchmarks that are to never be run with reference builds.
BENCHMARK_REF_BUILD_BLACKLIST = [
'power.idle_platform',
'power.idle_platform', # No browser used in benchmark.
'loading.desktop', # Long running benchmark.
'loading.mobile', # Long running benchmark.
'v8.runtime_stats.top_25', # Long running benchmark.
]
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment