Skip to content

GitLab

T
tangled
Commit 49c5485e authored by mikecase's avatar mikecase Committed by Commit bot
Browse files
Options

Add render test results to the results_details webpage. 

This will (1) make render tests perform pixel-wise
comparison and generate diff images, (2) allow you
to specify output directory for where to dump the
result images on the device, and (3) upload the results
to be visible from buildbot status page.

Review-Url: https://codereview.chromium.org/2866103002
Cr-Commit-Position: refs/heads/master@{#471144}
parent e0ecce46
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 360 additions and 331 deletions
build/android/pylib/instrumentation/render_test.html.jinja 0 → 100644
View file @ 49c5485e
<html>
<table>
<tr>
<th>Failure</th>
<th>Golden</th>
<th>Diff</th>
</tr>
<tr>
<td><img src="{{ failure_link }}" width="100%"/></td>
<td><img src="{{ golden_link }}" width="100%"/></td>
<td><img src="{{ diff_link }}" width="100%"/></td>
</tr>
</table>
</html>
build/android/pylib/local/device/local_device_instrumentation_test_run.py
View file @ 49c5485e
......@@ -6,6 +6,8 @@ import logging
import os
import posixpath
import re
import sys
import tempfile
import time
from devil.android import device_errors
......@@ -16,6 +18,7 @@ from devil.utils import reraiser_thread
from pylib import valgrind_tools
from pylib.android import logdog_logcat_monitor
from pylib.base import base_test_result
from pylib.constants import host_paths
from pylib.instrumentation import instrumentation_test_instance
from pylib.local.device import local_device_environment
from pylib.local.device import local_device_test_run
......@@ -26,6 +29,15 @@ from py_utils import contextlib_ext
from py_utils import tempfile_ext
import tombstones
sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
import jinja2 # pylint: disable=import-error
import markupsafe # pylint: disable=import-error,unused-import
_JINJA_TEMPLATE_DIR = os.path.join(
host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'instrumentation')
_JINJA_TEMPLATE_FILENAME = 'render_test.html.jinja'
_TAG = 'test_runner_py'
TIMEOUT_ANNOTATIONS = [
......@@ -43,6 +55,16 @@ LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v']
EXTRA_SCREENSHOT_FILE = (
'org.chromium.base.test.ScreenshotOnFailureStatement.ScreenshotFile')
FEATURE_ANNOTATION = 'Feature'
RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest'
# This needs to be kept in sync with formatting in |RenderUtils.imageName|
RE_RENDER_IMAGE_NAME = re.compile(
r'(?P<test_class>\w+)\.'
r'(?P<description>\w+)\.'
r'(?P<device_model>\w+)\.'
r'(?P<orientation>port|land)\.png')
# TODO(jbudorick): Make this private once the instrumentation test_runner is
# deprecated.
def DidPackageCrashOnDevice(package_name, device):
......@@ -249,7 +271,8 @@ class LocalDeviceInstrumentationTestRun(
def _RunTest(self, device, test):
extras = {}
flags = None
flags_to_add = []
flags_to_remove = []
test_timeout_scale = None
if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' % test[0]['method']
......@@ -304,7 +327,8 @@ class LocalDeviceInstrumentationTestRun(
self._test_instance.test_package, self._test_instance.test_runner)
extras['class'] = test_name
if 'flags' in test:
flags = test['flags']
flags_to_add.extend(test['flags'].add)
flags_to_remove.extend(test['flags'].remove)
timeout = self._GetTimeoutFromAnnotations(
test['annotations'], test_display_name)
......@@ -316,10 +340,19 @@ class LocalDeviceInstrumentationTestRun(
logging.info('preparing to run %s: %s', test_display_name, test)
if flags:
render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(),
'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add or flags_to_remove:
self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags(
add=flags.add, remove=flags.remove)
add=flags_to_add, remove=flags_to_remove)
try:
device.RunShellCommand(
......@@ -348,7 +381,7 @@ class LocalDeviceInstrumentationTestRun(
['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
check_return=True)
duration_ms = time_ms() - start_ms
if flags:
if flags_to_add or flags_to_remove:
self._flag_changers[str(device)].Restore()
if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale(
......@@ -364,8 +397,19 @@ class LocalDeviceInstrumentationTestRun(
if logcat_url:
result.SetLink('logcat', logcat_url)
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to check
# to see if any failure images were generated even if the test does not
# fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True, force=True)
# Update the result name if the test used flags.
if flags:
if flags_to_add or flags_to_remove:
for r in results:
if r.GetName() == test_name:
r.SetName(test_display_name)
......@@ -467,6 +511,97 @@ class LocalDeviceInstrumentationTestRun(
for result in results:
result.SetLink('post_test_screenshot', link)
def _ProcessRenderTestResults(
self, device, render_tests_device_output_dir, results):
# Will archive test images if we are given a GS bucket to store the results
# in and are given a results file to output the links to.
if not bool(self._test_instance.gs_results_bucket):
return
failure_images_device_dir = posixpath.join(
render_tests_device_output_dir, 'failures')
if not device.FileExists(failure_images_device_dir):
return
render_tests_bucket = (
self._test_instance.gs_results_bucket + '/render_tests')
diff_images_device_dir = posixpath.join(
render_tests_device_output_dir, 'diffs')
golden_images_device_dir = posixpath.join(
render_tests_device_output_dir, 'goldens')
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
device.PullFile(failure_images_device_dir, temp_dir)
if device.FileExists(diff_images_device_dir):
device.PullFile(diff_images_device_dir, temp_dir)
else:
logging.error('Diff images not found on device.')
if device.FileExists(golden_images_device_dir):
device.PullFile(golden_images_device_dir, temp_dir)
else:
logging.error('Golden images not found on device.')
for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')):
m = RE_RENDER_IMAGE_NAME.match(failure_filename)
if not m:
logging.warning('Unexpected file in render test failures: %s',
failure_filename)
continue
failure_filepath = os.path.join(temp_dir, 'failures', failure_filename)
failure_link = google_storage_helper.upload(
google_storage_helper.unique_name(
'failure_%s' % failure_filename, device=device),
failure_filepath,
bucket=render_tests_bucket)
golden_filepath = os.path.join(temp_dir, 'goldens', failure_filename)
if os.path.exists(golden_filepath):
golden_link = google_storage_helper.upload(
google_storage_helper.unique_name(
'golden_%s' % failure_filename, device=device),
golden_filepath,
bucket=render_tests_bucket)
else:
golden_link = ''
diff_filepath = os.path.join(temp_dir, 'diffs', failure_filename)
if os.path.exists(diff_filepath):
diff_link = google_storage_helper.upload(
google_storage_helper.unique_name(
'diff_%s' % failure_filename, device=device),
diff_filepath,
bucket=render_tests_bucket)
else:
diff_link = ''
with tempfile.NamedTemporaryFile(suffix='.html') as temp_html:
jinja2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
trim_blocks=True)
template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
# pylint: disable=no-member
processed_template_output = template.render(
failure_link=failure_link,
golden_link=golden_link,
diff_link=diff_link)
temp_html.write(processed_template_output)
temp_html.flush()
html_results_link = google_storage_helper.upload(
google_storage_helper.unique_name('render_html', device=device),
temp_html.name,
bucket=render_tests_bucket,
content_type='text/html')
for result in results:
result.SetLink(failure_filename, html_results_link)
#override
def _ShouldRetry(self, test):
if 'RetryOnFailure' in test.get('annotations', {}):
......@@ -502,3 +637,10 @@ class LocalDeviceInstrumentationTestRun(
timeout *= cls._GetTimeoutScaleFromAnnotations(annotations)
return timeout
def _IsRenderTest(test):
"""Determines if a test or list of tests has a RenderTest amongst them."""
if not isinstance(test, list):
test = [test]
return any([RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get(
FEATURE_ANNOTATION, ()) for t in test])
build/android/pylib/results/presentation/template/table.html
View file @ 49c5485e
......@@ -25,6 +25,9 @@
{%- elif cell.cell_type == 'links' -%}
{% for link in cell.links %}
<a href="{{link.href}}" target="{{link.target}}">{{link.data}}</a>
{% if not loop.last -%}
<br />
{%- endif %}
{% endfor %}
{%- elif cell.cell_type == 'action' -%}
<a onclick="{{cell.action}}">{{cell.data}}</a>
......
build/android/render_tests/process_render_test_results.py deleted 100755 → 0
View file @ e0ecce46
#!/usr/bin/env python
#
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import collections
import logging
import os
import posixpath
import re
import shutil
import sys
import tempfile
import zipfile
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
import devil_chromium
from devil.android import device_utils
from devil.utils import cmd_helper
from pylib.constants import host_paths
sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build'))
import find_depot_tools # pylint: disable=import-error
sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
import jinja2 # pylint: disable=import-error
try:
from PIL import Image # pylint: disable=import-error
from PIL import ImageChops # pylint: disable=import-error
can_compute_diffs = True
except ImportError:
can_compute_diffs = False
logging.exception('Error importing PIL library. Image diffs will not be '
'displayed properly unless PIL module is installed.')
_RE_IMAGE_NAME = re.compile(
r'(?P<test_class>\w+)\.'
r'(?P<description>\w+)\.'
r'(?P<device_model>\w+)\.'
r'(?P<orientation>port|land)\.png')
_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
def _UploadFiles(upload_dir, files):
"""Upload files to the render tests GS bucket."""
if files:
google_storage_upload_dir = os.path.join(_RENDER_TEST_BUCKET, upload_dir)
cmd = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'),
'-m', 'cp']
cmd.extend(files)
cmd.append(google_storage_upload_dir)
cmd_helper.RunCmd(cmd)
def _GoogleStorageUrl(upload_dir, filename):
return os.path.join(
_RENDER_TEST_BASE_URL, upload_dir, os.path.basename(filename))
def _ComputeImageDiff(failure_image, golden_image):
"""Compute mask showing which pixels are different between two images."""
return (ImageChops.difference(failure_image, golden_image)
.convert('L')
.point(lambda i: 255 if i else 0))
def ProcessRenderTestResults(devices, render_results_dir,
upload_dir, html_file):
"""Grabs render results from device and generates webpage displaying results.
Args:
devices: List of DeviceUtils objects to grab results from.
render_results_path: Path where render test results are storage.
Will look for failures render test results on the device in
/sdcard/chromium_tests_root/<render_results_path>/failures/
and will look for golden images at Chromium src/<render_results_path>/.
upload_dir: Directory to upload the render test results to.
html_file: File to write the test results to.
"""
results_dict = collections.defaultdict(lambda: collections.defaultdict(list))
diff_upload_dir = os.path.join(upload_dir, 'diffs')
failure_upload_dir = os.path.join(upload_dir, 'failures')
golden_upload_dir = os.path.join(upload_dir, 'goldens')
diff_images = []
failure_images = []
golden_images = []
temp_dir = None
try:
temp_dir = tempfile.mkdtemp()
for device in devices:
failures_device_dir = posixpath.join(
device.GetExternalStoragePath(),
'chromium_tests_root', render_results_dir, 'failures')
device.PullFile(failures_device_dir, temp_dir)
for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')):
m = _RE_IMAGE_NAME.match(failure_filename)
if not m:
logging.warning(
'Unexpected file in render test failures, %s', failure_filename)
continue
failure_file = os.path.join(temp_dir, 'failures', failure_filename)
# Check to make sure we have golden image for this failure.
golden_file = os.path.join(
host_paths.DIR_SOURCE_ROOT, render_results_dir, failure_filename)
if not os.path.exists(golden_file):
logging.error('Cannot find golden image for %s', failure_filename)
continue
# Compute image diff between failure and golden.
if can_compute_diffs:
diff_image = _ComputeImageDiff(
Image.open(failure_file), Image.open(golden_file))
diff_filename = '_diff'.join(
os.path.splitext(os.path.basename(failure_file)))
diff_file = os.path.join(temp_dir, diff_filename)
diff_image.save(diff_file)
diff_images.append(diff_file)
failure_images.append(failure_file)
golden_images.append(golden_file)
test_class = m.group('test_class')
device_model = m.group('device_model')
results_entry = {
'description': m.group('description'),
'orientation': m.group('orientation'),
'failure_image': _GoogleStorageUrl(failure_upload_dir, failure_file),
'golden_image': _GoogleStorageUrl(golden_upload_dir, golden_file),
}
if can_compute_diffs:
results_entry.update(
{'diff_image': _GoogleStorageUrl(diff_upload_dir, diff_file)})
results_dict[test_class][device_model].append(results_entry)
if can_compute_diffs:
_UploadFiles(diff_upload_dir, diff_images)
_UploadFiles(failure_upload_dir, failure_images)
_UploadFiles(golden_upload_dir, golden_images)
if failure_images:
failures_zipfile = os.path.join(temp_dir, 'failures.zip')
with zipfile.ZipFile(failures_zipfile, mode='w') as zf:
for failure_file in failure_images:
zf.write(failure_file, os.path.join(
render_results_dir, os.path.basename(failure_file)))
failure_zip_url = _GoogleStorageUrl(upload_dir, failures_zipfile)
_UploadFiles(upload_dir, [failures_zipfile])
else:
failure_zip_url = None
jinja2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
trim_blocks=True)
template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
# pylint: disable=no-member
processed_template_output = template.render(
full_results=dict(results_dict),
failure_zip_url=failure_zip_url, show_diffs=can_compute_diffs)
# pylint: enable=no-member
with open(html_file, 'wb') as f:
f.write(processed_template_output)
finally:
if temp_dir:
shutil.rmtree(temp_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--render-results-dir',
required=True,
help='Path on device to look for render test images')
parser.add_argument('--output-html-file',
required=True,
help='File to output the results webpage.')
parser.add_argument('-d', '--device', dest='devices', action='append',
default=[],
help='Device to look for render test results on. '
'Default is to look on all connected devices.')
parser.add_argument('--adb-path', type=os.path.abspath,
help='Absolute path to the adb binary to use.')
parser.add_argument('--buildername', type=str, required=True,
help='Bot buildername. Used to generate path to upload '
'render test results')
parser.add_argument('--build-number', type=str, required=True,
help='Bot build number. Used to generate path to upload '
'render test results')
args = parser.parse_args()
devil_chromium.Initialize(adb_path=args.adb_path)
devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
upload_dir = os.path.join(args.buildername, args.build_number)
ProcessRenderTestResults(
devices, args.render_results_dir, upload_dir, args.output_html_file)
if __name__ == '__main__':
sys.exit(main())
build/android/render_tests/render_webpage.html.jinja2 deleted 100644 → 0
View file @ e0ecce46
<!--
* Copyright 2016 The Chromium Authors. All rights reserved.
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
-->
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<link rel="stylesheet" href="https://code.getmdl.io/1.2.1/material.blue-indigo.min.css">
<script defer src="https://code.getmdl.io/1.2.1/material.min.js"></script>
<style>
div.text-element {
text-align: center;
}
body {
background-color: #efefef;
}
</style>
</head>
<body>
{% if failure_zip_url is not none %}
<a href="{{ failure_zip_url }}">
<div class="mdl-color--primary" width="100%">
<h3>Download Image Zip</h3>
</div>
</a>
{% endif %}
{% for test_class, device_results in full_results.iteritems() %}
<div class="mdl-color--primary" width="100%">
<h3>{{ test_class }}</h3>
</div>
<div class="mdl-tabs mdl-js-tabs mdl-js-ripple-effect">
<div class="mdl-tabs__tab-bar">
{% for device_model, _ in device_results.iteritems() %}
<a href="#{{ device_model }}-panel" class="mdl-tabs__tab">{{ device_model }}</a>
{% endfor %}
</div>
{% for device_model, test_results in device_results.iteritems() %}
<div class="mdl-tabs__panel" id="{{ device_model }}-panel">
<div class="mdl-grid">
<div class="mdl-cell mdl-cell--3-col text-element"><b>Description</b></div>
<div class="mdl-cell mdl-cell--3-col text-element"><b>Golden</b></div>
<div class="mdl-cell mdl-cell--3-col text-element"><b>Failure</b></div>
{% if show_diffs %}
<div class="mdl-cell mdl-cell--3-col text-element"><b>Diff</b></div>
{% endif %}
</div>
{% for result in test_results %}
<div class="mdl-grid">
<div class="mdl-cell mdl-cell--3-col text-element">
{{ result['description'] }}
</div>
<div class="mdl-cell mdl-cell--3-col">
<a href="{{ result['golden_image'] }}">
<img class="mdl-shadow--2dp" src="{{ result['golden_image'] }}" width="100%">
</a>
</div>
<div class="mdl-cell mdl-cell--3-col mdl-shadow--2dp">
<a href="{{ result['failure_image'] }}">
<img src="{{ result['failure_image'] }}" width="100%">
</a>
</div>
{% if show_diffs %}
<div class="mdl-cell mdl-cell--3-col mdl-shadow--2dp">
<a href="{{ result['diff_image'] }}">
<img src="{{ result['diff_image'] }}" width="100%">
</a>
</div>
{% endif %}
</div>
{% endfor %}
</div>
{% endfor %}
</div>
{% endfor %}
</body>
</html>
build/android/test_runner.pydeps
View file @ 49c5485e
......@@ -114,6 +114,26 @@
../../third_party/catapult/tracing/tracing_build/__init__.py
../../third_party/catapult/tracing/tracing_build/trace2html.py
../../third_party/catapult/tracing/tracing_project.py
../../third_party/jinja2/__init__.py
../../third_party/jinja2/_compat.py
../../third_party/jinja2/bccache.py
../../third_party/jinja2/compiler.py
../../third_party/jinja2/defaults.py
../../third_party/jinja2/environment.py
../../third_party/jinja2/exceptions.py
../../third_party/jinja2/filters.py
../../third_party/jinja2/lexer.py
../../third_party/jinja2/loaders.py
../../third_party/jinja2/nodes.py
../../third_party/jinja2/optimizer.py
../../third_party/jinja2/parser.py
../../third_party/jinja2/runtime.py
../../third_party/jinja2/tests.py
../../third_party/jinja2/utils.py
../../third_party/jinja2/visitor.py
../../third_party/markupsafe/__init__.py
../../third_party/markupsafe/_compat.py
../../third_party/markupsafe/_native.py
../../tools/swarming_client/libs/__init__.py
../../tools/swarming_client/libs/logdog/__init__.py
../../tools/swarming_client/libs/logdog/bootstrap.py
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment