Commit f43a1143 authored by kbr's avatar kbr Committed by Commit bot

Port hw_accel_feature and screenshot_sync tests to new harness.

BUG=352807
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:linux_optional_gpu_tests_rel;master.tryserver.chromium.mac:mac_optional_gpu_tests_rel;master.tryserver.chromium.win:win_optional_gpu_tests_rel;master.tryserver.chromium.android:android_optional_gpu_tests_rel
TBR=zmo@chromium.org
NOTRY=true

Review-Url: https://codereview.chromium.org/2619493004
Cr-Commit-Position: refs/heads/master@{#441956}
parent 6f439cbe
......@@ -1115,13 +1115,6 @@ NON_SWARMED_GTESTS = {
}
TELEMETRY_TESTS = {
'hardware_accelerated_feature': {
'tester_configs': [
{
'allow_on_android': True,
},
],
},
'maps_pixel_test': {
'target_name': 'maps',
'args': [
......@@ -1138,13 +1131,6 @@ TELEMETRY_TESTS = {
},
],
},
'screenshot_sync': {
'tester_configs': [
{
'allow_on_android': True,
},
],
},
}
# These tests use Telemetry's new, simpler, browser_test_runner.
......@@ -1173,6 +1159,13 @@ TELEMETRY_GPU_INTEGRATION_TESTS = {
}
],
},
'hardware_accelerated_feature': {
'tester_configs': [
{
'allow_on_android': True,
},
],
},
'pixel_test': {
'target_name': 'pixel',
'args': [
......@@ -1197,6 +1190,13 @@ TELEMETRY_GPU_INTEGRATION_TESTS = {
},
],
},
'screenshot_sync': {
'tester_configs': [
{
'allow_on_android': True,
},
],
},
'trace_test': {
'tester_configs': [
{
......
......@@ -9,5 +9,5 @@ from gpu_tests.gpu_test_expectations import GpuTestExpectations
class HardwareAcceleratedFeatureExpectations(GpuTestExpectations):
def SetExpectations(self):
# Accelerated 2D canvas is not available on Linux due to driver instability
self.Fail('HardwareAcceleratedFeature.canvas_accelerated',
self.Fail('HardwareAcceleratedFeature_canvas_accelerated',
['linux'], bug=254724)
# Copyright 2013 The Chromium Authors. All rights reserved.
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from gpu_tests import gpu_test_base
import gpu_tests.hardware_accelerated_feature_expectations as hw_expectations
from telemetry.page import legacy_page_test
from telemetry.story import story_set as story_set_module
from gpu_tests import gpu_integration_test
import gpu_tests.hardware_accelerated_feature_expectations as hw_expectations
test_harness_script = r"""
function VerifyHardwareAccelerated(feature) {
......@@ -24,44 +22,51 @@ test_harness_script = r"""
};
"""
class HardwareAcceleratedFeatureValidator(gpu_test_base.ValidatorBase):
def ValidateAndMeasurePage(self, page, tab, results):
feature = page.feature
if not tab.EvaluateJavaScript('VerifyHardwareAccelerated("%s")' % feature):
print 'Test failed. Printing page contents:'
print tab.EvaluateJavaScript('document.body.innerHTML')
raise legacy_page_test.Failure('%s not hardware accelerated' % feature)
def safe_feature_name(feature):
return feature.lower().replace(' ', '_')
class ChromeGpuPage(gpu_test_base.PageBase):
def __init__(self, story_set, feature, expectations):
super(ChromeGpuPage, self).__init__(
url='chrome://gpu', page_set=story_set, base_dir=story_set.base_dir,
name=('HardwareAcceleratedFeature.%s_accelerated' %
safe_feature_name(feature)),
expectations=expectations)
self.feature = feature
self.script_to_evaluate_on_commit = test_harness_script
class HardwareAcceleratedFeature(gpu_test_base.TestBase):
"""Tests GPU acceleration is reported as active for various features"""
test = HardwareAcceleratedFeatureValidator
class HardwareAcceleratedFeatureIntegrationTest(
gpu_integration_test.GpuIntegrationTest):
"""Tests GPU acceleration is reported as active for various features."""
@classmethod
def Name(cls):
"""The name by which this test is invoked on the command line."""
return 'hardware_accelerated_feature'
def _CreateExpectations(self):
@classmethod
def setUpClass(cls):
super(cls, HardwareAcceleratedFeatureIntegrationTest).setUpClass()
cls.SetBrowserOptions(cls._finder_options)
cls.StartBrowser()
cls.SetStaticServerDirs([])
@classmethod
def _CreateExpectations(cls):
return hw_expectations.HardwareAcceleratedFeatureExpectations()
def CreateStorySet(self, options):
features = ['WebGL', 'Canvas']
def _Navigate(self, url):
# It's crucial to use the action_runner, rather than the tab's
# Navigate method directly. It waits for the document ready state
# to become interactive or better, avoiding critical race
# conditions.
self.tab.action_runner.Navigate(
url, script_to_evaluate_on_commit=test_harness_script)
ps = story_set_module.StorySet()
@classmethod
def GenerateGpuTests(cls, options):
tests = ('WebGL', 'Canvas')
for feature in tests:
yield('HardwareAcceleratedFeature_%s_accelerated' %
safe_feature_name(feature),
'chrome://gpu',
(feature))
for feature in features:
ps.AddStory(ChromeGpuPage(story_set=ps, feature=feature,
expectations=self.GetExpectations()))
return ps
def RunActualGpuTest(self, test_path, *args):
feature = args[0]
self._Navigate(test_path)
tab = self.tab
if not tab.EvaluateJavaScript('VerifyHardwareAccelerated("%s")' % feature):
print 'Test failed. Printing page contents:'
print tab.EvaluateJavaScript('document.body.innerHTML')
self.fail('%s not hardware accelerated' % feature)
# Copyright 2014 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import os
import random
from gpu_tests import gpu_test_base
from gpu_tests import path_util
from gpu_tests import screenshot_sync_expectations
from telemetry.page import legacy_page_test
from telemetry.story import story_set as story_set_module
from telemetry.util import image_util
from telemetry.util import rgba_color
data_path = os.path.join(
path_util.GetChromiumSrcDir(), 'content', 'test', 'data', 'gpu')
class SoftwareRasterSharedPageState(gpu_test_base.GpuSharedPageState):
def __init__(self, test, finder_options, story_set):
super(SoftwareRasterSharedPageState, self).__init__(
test, finder_options, story_set)
finder_options.browser_options.AppendExtraBrowserArgs(
['--disable-gpu-rasterization'])
class GPURasterSharedPageState(gpu_test_base.GpuSharedPageState):
def __init__(self, test, finder_options, story_set):
super(GPURasterSharedPageState, self).__init__(
test, finder_options, story_set)
finder_options.browser_options.AppendExtraBrowserArgs(
['--force-gpu-rasterization'])
class ScreenshotSyncValidator(gpu_test_base.ValidatorBase):
def CustomizeBrowserOptions(self, options):
# --test-type=gpu is used only to suppress the "Google API Keys are missing"
# infobar, which causes flakiness in tests.
options.AppendExtraBrowserArgs(['--test-type=gpu'])
def ValidateAndMeasurePage(self, page, tab, results):
if not tab.screenshot_supported:
raise legacy_page_test.Failure(
'Browser does not support screenshot capture')
def CheckColorMatchAtLocation(expectedRGB, screenshot, x, y):
pixel_value = image_util.GetPixelColor(screenshot, x, y)
if not expectedRGB.IsEqual(pixel_value):
error_message = ('Color mismatch at (%d, %d): expected (%d, %d, %d), ' +
'got (%d, %d, %d)') % (
x, y, expectedRGB.r, expectedRGB.g, expectedRGB.b,
pixel_value.r, pixel_value.g, pixel_value.b)
raise legacy_page_test.Failure(error_message)
def CheckScreenshot():
canvasRGB = rgba_color.RgbaColor(random.randint(0, 255),
random.randint(0, 255),
random.randint(0, 255),
255)
tab.EvaluateJavaScript("window.draw(%d, %d, %d);" % (
canvasRGB.r, canvasRGB.g, canvasRGB.b))
screenshot = tab.Screenshot(5)
start_x = 10
start_y = 0
outer_size = 256
skip = 10
for y in range(start_y, outer_size, skip):
for x in range(start_x, outer_size, skip):
CheckColorMatchAtLocation(canvasRGB, screenshot, x, y)
repetitions = 20
for _ in range(0, repetitions):
CheckScreenshot()
class ScreenshotSyncPage(gpu_test_base.PageBase):
def __init__(self, story_set, base_dir,
shared_page_state_class,
url, name, expectations):
super(ScreenshotSyncPage, self).__init__(
url=url,
page_set=story_set,
base_dir=base_dir,
name=name,
shared_page_state_class=shared_page_state_class,
expectations=expectations)
class ScreenshotSyncStorySet(story_set_module.StorySet):
"""Test cases for screenshots being in sync with content updates."""
def __init__(self, base_dir=None, serving_dirs=None):
super(ScreenshotSyncStorySet, self).__init__(
base_dir=base_dir, serving_dirs=serving_dirs)
@property
def allow_mixed_story_states(self):
# Return True here in order to be able to run the same tests with
# both software and GPU rasterization.
return True
class ScreenshotSyncProcess(gpu_test_base.TestBase):
"""Tests that screenhots are properly synchronized with the frame one which
they were requested"""
test = ScreenshotSyncValidator
@classmethod
def Name(cls):
return 'screenshot_sync'
def _CreateExpectations(self):
return screenshot_sync_expectations.ScreenshotSyncExpectations()
def CreateStorySet(self, options):
ps = ScreenshotSyncStorySet(base_dir=data_path, serving_dirs=[''])
ps.AddStory(ScreenshotSyncPage(ps, ps.base_dir,
SoftwareRasterSharedPageState,
'file://screenshot_sync_canvas.html',
'ScreenshotSync.SWRasterWithCanvas',
self.GetExpectations()))
ps.AddStory(ScreenshotSyncPage(ps, ps.base_dir,
SoftwareRasterSharedPageState,
'file://screenshot_sync_divs.html',
'ScreenshotSync.SWRasterWithDivs',
self.GetExpectations()))
ps.AddStory(ScreenshotSyncPage(ps, ps.base_dir,
GPURasterSharedPageState,
'file://screenshot_sync_canvas.html',
'ScreenshotSync.GPURasterWithCanvas',
self.GetExpectations()))
ps.AddStory(ScreenshotSyncPage(ps, ps.base_dir,
GPURasterSharedPageState,
'file://screenshot_sync_divs.html',
'ScreenshotSync.GPURasterWithDivs',
self.GetExpectations()))
return ps
......@@ -11,13 +11,13 @@ class ScreenshotSyncExpectations(GpuTestExpectations):
super(ScreenshotSyncExpectations, self).__init__(*args, **kwargs)
def SetExpectations(self):
self.Flaky('ScreenshotSync.GPURasterWithCanvas', ['win', 'amd'],
self.Flaky('ScreenshotSync_GPURasterWithCanvas', ['win', 'amd'],
bug=599776)
self.Flaky('ScreenshotSync.GPURasterWithCanvas', ['mac', 'intel'],
self.Flaky('ScreenshotSync_GPURasterWithCanvas', ['mac', 'intel'],
bug=599776)
self.Flaky('ScreenshotSync.GPURasterWithDivs', ['mac', 'intel'],
self.Flaky('ScreenshotSync_GPURasterWithDivs', ['mac', 'intel'],
bug=599776)
self.Fail('ScreenshotSync.GPURasterWithDivs', ['win', ('amd', 0x6613)],
self.Fail('ScreenshotSync_GPURasterWithDivs', ['win', ('amd', 0x6613)],
bug=639489)
# TODO(kbr): flakily timing out on this configuration.
self.Flaky('*', ['linux', 'intel', 'debug'], bug=648369)
# Copyright 2017 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging
import os
import random
from gpu_tests import gpu_integration_test
from gpu_tests import path_util
from gpu_tests import screenshot_sync_expectations
from telemetry.util import image_util
from telemetry.util import rgba_color
data_path = os.path.join(
path_util.GetChromiumSrcDir(), 'content', 'test', 'data', 'gpu')
class ScreenshotSyncIntegrationTest(gpu_integration_test.GpuIntegrationTest):
"""Tests that screenshots are properly synchronized with the frame on
which they were requested.
"""
# We store a deep copy of the original browser finder options in
# order to be able to restart the browser multiple times, with a
# different set of command line arguments each time.
_original_finder_options = None
# We keep track of the set of command line arguments used to launch
# the browser most recently in order to figure out whether we need
# to relaunch it, if a new pixel test requires a different set of
# arguments.
_last_launched_browser_args = set()
@classmethod
def Name(cls):
"""The name by which this test is invoked on the command line."""
return 'screenshot_sync'
@classmethod
def setUpClass(cls):
super(cls, ScreenshotSyncIntegrationTest).setUpClass()
cls._original_finder_options = cls._finder_options.Copy()
cls.CustomizeBrowserArgs([])
cls.StartBrowser()
cls.SetStaticServerDirs([data_path])
@classmethod
def CustomizeBrowserArgs(cls, browser_args):
if not browser_args:
browser_args = []
cls._finder_options = cls._original_finder_options.Copy()
browser_options = cls._finder_options.browser_options
# All tests receive the following options. They aren't recorded in
# the _last_launched_browser_args.
#
# --test-type=gpu is used only to suppress the "Google API Keys are missing"
# infobar, which causes flakiness in tests.
browser_options.AppendExtraBrowserArgs(['--test-type=gpu'])
# Append the new arguments.
browser_options.AppendExtraBrowserArgs(browser_args)
cls._last_launched_browser_args = set(browser_args)
cls.SetBrowserOptions(cls._finder_options)
@classmethod
def RestartBrowserIfNecessaryWithArgs(cls, browser_args):
if not browser_args:
browser_args = []
if set(browser_args) != cls._last_launched_browser_args:
logging.info('Restarting browser with arguments: ' + str(browser_args))
cls.StopBrowser()
cls.CustomizeBrowserArgs(browser_args)
cls.StartBrowser()
cls.tab = cls.browser.tabs[0]
@classmethod
def _CreateExpectations(cls):
return screenshot_sync_expectations.ScreenshotSyncExpectations()
@classmethod
def GenerateGpuTests(cls, options):
yield('ScreenshotSync_SWRasterWithCanvas',
'screenshot_sync_canvas.html',
('--disable-gpu-rasterization'))
yield('ScreenshotSync_SWRasterWithDivs',
'screenshot_sync_divs.html',
('--disable-gpu-rasterization'))
yield('ScreenshotSync_GPURasterWithCanvas',
'screenshot_sync_canvas.html',
('--force-gpu-rasterization'))
yield('ScreenshotSync_GPURasterWithDivs',
'screenshot_sync_divs.html',
('--force-gpu-rasterization'))
def _Navigate(self, test_path):
url = self.UrlOfStaticFilePath(test_path)
# It's crucial to use the action_runner, rather than the tab's
# Navigate method directly. It waits for the document ready state
# to become interactive or better, avoiding critical race
# conditions.
self.tab.action_runner.Navigate(url)
def _CheckColorMatchAtLocation(self, expectedRGB, screenshot, x, y):
pixel_value = image_util.GetPixelColor(screenshot, x, y)
if not expectedRGB.IsEqual(pixel_value):
error_message = ('Color mismatch at (%d, %d): expected (%d, %d, %d), ' +
'got (%d, %d, %d)') % (
x, y, expectedRGB.r, expectedRGB.g, expectedRGB.b,
pixel_value.r, pixel_value.g, pixel_value.b)
self.fail(error_message)
def _CheckScreenshot(self):
canvasRGB = rgba_color.RgbaColor(random.randint(0, 255),
random.randint(0, 255),
random.randint(0, 255),
255)
tab = self.tab
tab.EvaluateJavaScript("window.draw(%d, %d, %d);" % (
canvasRGB.r, canvasRGB.g, canvasRGB.b))
screenshot = tab.Screenshot(5)
start_x = 10
start_y = 0
outer_size = 256
skip = 10
for y in range(start_y, outer_size, skip):
for x in range(start_x, outer_size, skip):
self._CheckColorMatchAtLocation(canvasRGB, screenshot, x, y)
def RunActualGpuTest(self, test_path, *args):
browser_arg = args[0]
self.RestartBrowserIfNecessaryWithArgs([browser_arg])
self._Navigate(test_path)
repetitions = 20
for _ in range(0, repetitions):
self._CheckScreenshot()
This diff is collapsed.
......@@ -137,10 +137,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -226,10 +226,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -435,10 +435,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -524,10 +524,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -717,10 +717,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -806,10 +806,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1015,10 +1015,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1104,10 +1104,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1303,10 +1303,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1395,10 +1395,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1614,10 +1614,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1706,10 +1706,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1902,10 +1902,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -1991,10 +1991,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -2200,10 +2200,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "hardware_accelerated_feature_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......@@ -2289,10 +2289,10 @@
"-v",
"--extra-browser-args=--enable-logging=stderr --js-flags=--expose-gc"
],
"isolate_name": "telemetry_gpu_test",
"isolate_name": "telemetry_gpu_integration_test",
"name": "screenshot_sync_tests",
"override_compile_targets": [
"telemetry_gpu_test_run"
"telemetry_gpu_integration_test_run"
],
"swarming": {
"can_use_on_swarming_builders": true,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment