Skip to content

GitLab

T
tangled
Commit 49c5485e authored by mikecase's avatar mikecase Committed by Commit bot
Browse files
Options

Add render test results to the results_details webpage. 

This will (1) make render tests perform pixel-wise
comparison and generate diff images, (2) allow you
to specify output directory for where to dump the
result images on the device, and (3) upload the results
to be visible from buildbot status page.

Review-Url: https://codereview.chromium.org/2866103002
Cr-Commit-Position: refs/heads/master@{#471144}
parent e0ecce46
Show whitespace changes
Inline Side-by-side
Showing with 360 additions and 331 deletions
build/android/pylib/instrumentation/render_test.html.jinja 0 → 100644
View file @ 49c5485e
<html>
<table>
<tr>
<th>Failure</th>
<th>Golden</th>
<th>Diff</th>
</tr>
<tr>
<td><img src="{{ failure_link }}" width="100%"/></td>
<td><img src="{{ golden_link }}" width="100%"/></td>
<td><img src="{{ diff_link }}" width="100%"/></td>
</tr>
</table>
</html>
build/android/pylib/local/device/local_device_instrumentation_test_run.py
View file @ 49c5485e
...@@ -6,6 +6,8 @@ import logging ...@@ -6,6 +6,8 @@ import logging
import os import os
import posixpath import posixpath
import re import re
import sys
import tempfile
import time import time
from devil.android import device_errors from devil.android import device_errors
...@@ -16,6 +18,7 @@ from devil.utils import reraiser_thread ...@@ -16,6 +18,7 @@ from devil.utils import reraiser_thread
from pylib import valgrind_tools from pylib import valgrind_tools
from pylib.android import logdog_logcat_monitor from pylib.android import logdog_logcat_monitor
from pylib.base import base_test_result from pylib.base import base_test_result
from pylib.constants import host_paths
from pylib.instrumentation import instrumentation_test_instance from pylib.instrumentation import instrumentation_test_instance
from pylib.local.device import local_device_environment from pylib.local.device import local_device_environment
from pylib.local.device import local_device_test_run from pylib.local.device import local_device_test_run
...@@ -26,6 +29,15 @@ from py_utils import contextlib_ext ...@@ -26,6 +29,15 @@ from py_utils import contextlib_ext
from py_utils import tempfile_ext from py_utils import tempfile_ext
import tombstones import tombstones
sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
import jinja2 # pylint: disable=import-error
import markupsafe # pylint: disable=import-error,unused-import
_JINJA_TEMPLATE_DIR = os.path.join(
host_paths.DIR_SOURCE_ROOT, 'build', 'android', 'pylib', 'instrumentation')
_JINJA_TEMPLATE_FILENAME = 'render_test.html.jinja'
_TAG = 'test_runner_py' _TAG = 'test_runner_py'
TIMEOUT_ANNOTATIONS = [ TIMEOUT_ANNOTATIONS = [
...@@ -43,6 +55,16 @@ LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v'] ...@@ -43,6 +55,16 @@ LOGCAT_FILTERS = ['*:e', 'chromium:v', 'cr_*:v']
EXTRA_SCREENSHOT_FILE = ( EXTRA_SCREENSHOT_FILE = (
'org.chromium.base.test.ScreenshotOnFailureStatement.ScreenshotFile') 'org.chromium.base.test.ScreenshotOnFailureStatement.ScreenshotFile')
FEATURE_ANNOTATION = 'Feature'
RENDER_TEST_FEATURE_ANNOTATION = 'RenderTest'
# This needs to be kept in sync with formatting in |RenderUtils.imageName|
RE_RENDER_IMAGE_NAME = re.compile(
r'(?P<test_class>\w+)\.'
r'(?P<description>\w+)\.'
r'(?P<device_model>\w+)\.'
r'(?P<orientation>port|land)\.png')
# TODO(jbudorick): Make this private once the instrumentation test_runner is # TODO(jbudorick): Make this private once the instrumentation test_runner is
# deprecated. # deprecated.
def DidPackageCrashOnDevice(package_name, device): def DidPackageCrashOnDevice(package_name, device):
...@@ -249,7 +271,8 @@ class LocalDeviceInstrumentationTestRun( ...@@ -249,7 +271,8 @@ class LocalDeviceInstrumentationTestRun(
def _RunTest(self, device, test): def _RunTest(self, device, test):
extras = {} extras = {}
flags = None flags_to_add = []
flags_to_remove = []
test_timeout_scale = None test_timeout_scale = None
if self._test_instance.coverage_directory: if self._test_instance.coverage_directory:
coverage_basename = '%s.ec' % ('%s_group' % test[0]['method'] coverage_basename = '%s.ec' % ('%s_group' % test[0]['method']
...@@ -304,7 +327,8 @@ class LocalDeviceInstrumentationTestRun( ...@@ -304,7 +327,8 @@ class LocalDeviceInstrumentationTestRun(
self._test_instance.test_package, self._test_instance.test_runner) self._test_instance.test_package, self._test_instance.test_runner)
extras['class'] = test_name extras['class'] = test_name
if 'flags' in test: if 'flags' in test:
flags = test['flags'] flags_to_add.extend(test['flags'].add)
flags_to_remove.extend(test['flags'].remove)
timeout = self._GetTimeoutFromAnnotations( timeout = self._GetTimeoutFromAnnotations(
test['annotations'], test_display_name) test['annotations'], test_display_name)
...@@ -316,10 +340,19 @@ class LocalDeviceInstrumentationTestRun( ...@@ -316,10 +340,19 @@ class LocalDeviceInstrumentationTestRun(
logging.info('preparing to run %s: %s', test_display_name, test) logging.info('preparing to run %s: %s', test_display_name, test)
if flags: render_tests_device_output_dir = None
if _IsRenderTest(test):
# TODO(mikecase): Add DeviceTempDirectory class and use that instead.
render_tests_device_output_dir = posixpath.join(
device.GetExternalStoragePath(),
'render_test_output_dir')
flags_to_add.append('--render-test-output-dir=%s' %
render_tests_device_output_dir)
if flags_to_add or flags_to_remove:
self._CreateFlagChangerIfNeeded(device) self._CreateFlagChangerIfNeeded(device)
self._flag_changers[str(device)].PushFlags( self._flag_changers[str(device)].PushFlags(
add=flags.add, remove=flags.remove) add=flags_to_add, remove=flags_to_remove)
try: try:
device.RunShellCommand( device.RunShellCommand(
...@@ -348,7 +381,7 @@ class LocalDeviceInstrumentationTestRun( ...@@ -348,7 +381,7 @@ class LocalDeviceInstrumentationTestRun(
['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name], ['log', '-p', 'i', '-t', _TAG, 'END %s' % test_name],
check_return=True) check_return=True)
duration_ms = time_ms() - start_ms duration_ms = time_ms() - start_ms
if flags: if flags_to_add or flags_to_remove:
self._flag_changers[str(device)].Restore() self._flag_changers[str(device)].Restore()
if test_timeout_scale: if test_timeout_scale:
valgrind_tools.SetChromeTimeoutScale( valgrind_tools.SetChromeTimeoutScale(
...@@ -364,8 +397,19 @@ class LocalDeviceInstrumentationTestRun( ...@@ -364,8 +397,19 @@ class LocalDeviceInstrumentationTestRun(
if logcat_url: if logcat_url:
result.SetLink('logcat', logcat_url) result.SetLink('logcat', logcat_url)
if _IsRenderTest(test):
# Render tests do not cause test failure by default. So we have to check
# to see if any failure images were generated even if the test does not
# fail.
try:
self._ProcessRenderTestResults(
device, render_tests_device_output_dir, results)
finally:
device.RemovePath(render_tests_device_output_dir,
recursive=True, force=True)
# Update the result name if the test used flags. # Update the result name if the test used flags.
if flags: if flags_to_add or flags_to_remove:
for r in results: for r in results:
if r.GetName() == test_name: if r.GetName() == test_name:
r.SetName(test_display_name) r.SetName(test_display_name)
...@@ -467,6 +511,97 @@ class LocalDeviceInstrumentationTestRun( ...@@ -467,6 +511,97 @@ class LocalDeviceInstrumentationTestRun(
for result in results: for result in results:
result.SetLink('post_test_screenshot', link) result.SetLink('post_test_screenshot', link)
def _ProcessRenderTestResults(
self, device, render_tests_device_output_dir, results):
# Will archive test images if we are given a GS bucket to store the results
# in and are given a results file to output the links to.
if not bool(self._test_instance.gs_results_bucket):
return
failure_images_device_dir = posixpath.join(
render_tests_device_output_dir, 'failures')
if not device.FileExists(failure_images_device_dir):
return
render_tests_bucket = (
self._test_instance.gs_results_bucket + '/render_tests')
diff_images_device_dir = posixpath.join(
render_tests_device_output_dir, 'diffs')
golden_images_device_dir = posixpath.join(
render_tests_device_output_dir, 'goldens')
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
device.PullFile(failure_images_device_dir, temp_dir)
if device.FileExists(diff_images_device_dir):
device.PullFile(diff_images_device_dir, temp_dir)
else:
logging.error('Diff images not found on device.')
if device.FileExists(golden_images_device_dir):
device.PullFile(golden_images_device_dir, temp_dir)
else:
logging.error('Golden images not found on device.')
for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')):
m = RE_RENDER_IMAGE_NAME.match(failure_filename)
if not m:
logging.warning('Unexpected file in render test failures: %s',
failure_filename)
continue
failure_filepath = os.path.join(temp_dir, 'failures', failure_filename)
failure_link = google_storage_helper.upload(
google_storage_helper.unique_name(
'failure_%s' % failure_filename, device=device),
failure_filepath,
bucket=render_tests_bucket)
golden_filepath = os.path.join(temp_dir, 'goldens', failure_filename)
if os.path.exists(golden_filepath):
golden_link = google_storage_helper.upload(
google_storage_helper.unique_name(
'golden_%s' % failure_filename, device=device),
golden_filepath,
bucket=render_tests_bucket)
else:
golden_link = ''
diff_filepath = os.path.join(temp_dir, 'diffs', failure_filename)
if os.path.exists(diff_filepath):
diff_link = google_storage_helper.upload(
google_storage_helper.unique_name(
'diff_%s' % failure_filename, device=device),
diff_filepath,
bucket=render_tests_bucket)
else:
diff_link = ''
with tempfile.NamedTemporaryFile(suffix='.html') as temp_html:
jinja2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
trim_blocks=True)
template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
# pylint: disable=no-member
processed_template_output = template.render(
failure_link=failure_link,
golden_link=golden_link,
diff_link=diff_link)
temp_html.write(processed_template_output)
temp_html.flush()
html_results_link = google_storage_helper.upload(
google_storage_helper.unique_name('render_html', device=device),
temp_html.name,
bucket=render_tests_bucket,
content_type='text/html')
for result in results:
result.SetLink(failure_filename, html_results_link)
#override #override
def _ShouldRetry(self, test): def _ShouldRetry(self, test):
if 'RetryOnFailure' in test.get('annotations', {}): if 'RetryOnFailure' in test.get('annotations', {}):
...@@ -502,3 +637,10 @@ class LocalDeviceInstrumentationTestRun( ...@@ -502,3 +637,10 @@ class LocalDeviceInstrumentationTestRun(
timeout *= cls._GetTimeoutScaleFromAnnotations(annotations) timeout *= cls._GetTimeoutScaleFromAnnotations(annotations)
return timeout return timeout
def _IsRenderTest(test):
"""Determines if a test or list of tests has a RenderTest amongst them."""
if not isinstance(test, list):
test = [test]
return any([RENDER_TEST_FEATURE_ANNOTATION in t['annotations'].get(
FEATURE_ANNOTATION, ()) for t in test])
build/android/pylib/results/presentation/template/table.html
View file @ 49c5485e
...@@ -25,6 +25,9 @@ ...@@ -25,6 +25,9 @@
{%- elif cell.cell_type == 'links' -%} {%- elif cell.cell_type == 'links' -%}
{% for link in cell.links %} {% for link in cell.links %}
<a href="{{link.href}}" target="{{link.target}}">{{link.data}}</a> <a href="{{link.href}}" target="{{link.target}}">{{link.data}}</a>
{% if not loop.last -%}
<br />
{%- endif %}
{% endfor %} {% endfor %}
{%- elif cell.cell_type == 'action' -%} {%- elif cell.cell_type == 'action' -%}
<a onclick="{{cell.action}}">{{cell.data}}</a> <a onclick="{{cell.action}}">{{cell.data}}</a>
......
build/android/render_tests/process_render_test_results.py deleted 100755 → 0
View file @ e0ecce46
#!/usr/bin/env python
#
# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
import collections
import logging
import os
import posixpath
import re
import shutil
import sys
import tempfile
import zipfile
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
import devil_chromium
from devil.android import device_utils
from devil.utils import cmd_helper
from pylib.constants import host_paths
sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'build'))
import find_depot_tools # pylint: disable=import-error
sys.path.append(os.path.join(host_paths.DIR_SOURCE_ROOT, 'third_party'))
import jinja2 # pylint: disable=import-error
try:
from PIL import Image # pylint: disable=import-error
from PIL import ImageChops # pylint: disable=import-error
can_compute_diffs = True
except ImportError:
can_compute_diffs = False
logging.exception('Error importing PIL library. Image diffs will not be '
'displayed properly unless PIL module is installed.')
_RE_IMAGE_NAME = re.compile(
r'(?P<test_class>\w+)\.'
r'(?P<description>\w+)\.'
r'(?P<device_model>\w+)\.'
r'(?P<orientation>port|land)\.png')
_RENDER_TEST_BASE_URL = 'https://storage.googleapis.com/chromium-render-tests/'
_RENDER_TEST_BUCKET = 'gs://chromium-render-tests/'
_JINJA_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
_JINJA_TEMPLATE_FILENAME = 'render_webpage.html.jinja2'
def _UploadFiles(upload_dir, files):
"""Upload files to the render tests GS bucket."""
if files:
google_storage_upload_dir = os.path.join(_RENDER_TEST_BUCKET, upload_dir)
cmd = [os.path.join(find_depot_tools.DEPOT_TOOLS_PATH, 'gsutil.py'),
'-m', 'cp']
cmd.extend(files)
cmd.append(google_storage_upload_dir)
cmd_helper.RunCmd(cmd)
def _GoogleStorageUrl(upload_dir, filename):
return os.path.join(
_RENDER_TEST_BASE_URL, upload_dir, os.path.basename(filename))
def _ComputeImageDiff(failure_image, golden_image):
"""Compute mask showing which pixels are different between two images."""
return (ImageChops.difference(failure_image, golden_image)
.convert('L')
.point(lambda i: 255 if i else 0))
def ProcessRenderTestResults(devices, render_results_dir,
upload_dir, html_file):
"""Grabs render results from device and generates webpage displaying results.
Args:
devices: List of DeviceUtils objects to grab results from.
render_results_path: Path where render test results are storage.
Will look for failures render test results on the device in
/sdcard/chromium_tests_root/<render_results_path>/failures/
and will look for golden images at Chromium src/<render_results_path>/.
upload_dir: Directory to upload the render test results to.
html_file: File to write the test results to.
"""
results_dict = collections.defaultdict(lambda: collections.defaultdict(list))
diff_upload_dir = os.path.join(upload_dir, 'diffs')
failure_upload_dir = os.path.join(upload_dir, 'failures')
golden_upload_dir = os.path.join(upload_dir, 'goldens')
diff_images = []
failure_images = []
golden_images = []
temp_dir = None
try:
temp_dir = tempfile.mkdtemp()
for device in devices:
failures_device_dir = posixpath.join(
device.GetExternalStoragePath(),
'chromium_tests_root', render_results_dir, 'failures')
device.PullFile(failures_device_dir, temp_dir)
for failure_filename in os.listdir(os.path.join(temp_dir, 'failures')):
m = _RE_IMAGE_NAME.match(failure_filename)
if not m:
logging.warning(
'Unexpected file in render test failures, %s', failure_filename)
continue
failure_file = os.path.join(temp_dir, 'failures', failure_filename)
# Check to make sure we have golden image for this failure.
golden_file = os.path.join(
host_paths.DIR_SOURCE_ROOT, render_results_dir, failure_filename)
if not os.path.exists(golden_file):
logging.error('Cannot find golden image for %s', failure_filename)
continue
# Compute image diff between failure and golden.
if can_compute_diffs:
diff_image = _ComputeImageDiff(
Image.open(failure_file), Image.open(golden_file))
diff_filename = '_diff'.join(
os.path.splitext(os.path.basename(failure_file)))
diff_file = os.path.join(temp_dir, diff_filename)
diff_image.save(diff_file)
diff_images.append(diff_file)
failure_images.append(failure_file)
golden_images.append(golden_file)
test_class = m.group('test_class')
device_model = m.group('device_model')
results_entry = {
'description': m.group('description'),
'orientation': m.group('orientation'),
'failure_image': _GoogleStorageUrl(failure_upload_dir, failure_file),
'golden_image': _GoogleStorageUrl(golden_upload_dir, golden_file),
}
if can_compute_diffs:
results_entry.update(
{'diff_image': _GoogleStorageUrl(diff_upload_dir, diff_file)})
results_dict[test_class][device_model].append(results_entry)
if can_compute_diffs:
_UploadFiles(diff_upload_dir, diff_images)
_UploadFiles(failure_upload_dir, failure_images)
_UploadFiles(golden_upload_dir, golden_images)
if failure_images:
failures_zipfile = os.path.join(temp_dir, 'failures.zip')
with zipfile.ZipFile(failures_zipfile, mode='w') as zf:
for failure_file in failure_images:
zf.write(failure_file, os.path.join(
render_results_dir, os.path.basename(failure_file)))
failure_zip_url = _GoogleStorageUrl(upload_dir, failures_zipfile)
_UploadFiles(upload_dir, [failures_zipfile])
else:
failure_zip_url = None
jinja2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(_JINJA_TEMPLATE_DIR),
trim_blocks=True)
template = jinja2_env.get_template(_JINJA_TEMPLATE_FILENAME)
# pylint: disable=no-member
processed_template_output = template.render(
full_results=dict(results_dict),
failure_zip_url=failure_zip_url, show_diffs=can_compute_diffs)
# pylint: enable=no-member
with open(html_file, 'wb') as f:
f.write(processed_template_output)
finally:
if temp_dir:
shutil.rmtree(temp_dir)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--render-results-dir',
required=True,
help='Path on device to look for render test images')
parser.add_argument('--output-html-file',
required=True,
help='File to output the results webpage.')
parser.add_argument('-d', '--device', dest='devices', action='append',
default=[],
help='Device to look for render test results on. '
'Default is to look on all connected devices.')
parser.add_argument('--adb-path', type=os.path.abspath,
help='Absolute path to the adb binary to use.')
parser.add_argument('--buildername', type=str, required=True,
help='Bot buildername. Used to generate path to upload '
'render test results')
parser.add_argument('--build-number', type=str, required=True,
help='Bot build number. Used to generate path to upload '
'render test results')
args = parser.parse_args()
devil_chromium.Initialize(adb_path=args.adb_path)
devices = device_utils.DeviceUtils.HealthyDevices(device_arg=args.devices)
upload_dir = os.path.join(args.buildername, args.build_number)
ProcessRenderTestResults(
devices, args.render_results_dir, upload_dir, args.output_html_file)
if __name__ == '__main__':
sys.exit(main())
build/android/render_tests/render_webpage.html.jinja2 deleted 100644 → 0
View file @ e0ecce46
<!--
* Copyright 2016 The Chromium Authors. All rights reserved.
* Use of this source code is governed by a BSD-style license that can be
* found in the LICENSE file.
-->
<!DOCTYPE html>
<html>
<head>
<link rel="stylesheet" href="https://fonts.googleapis.com/icon?family=Material+Icons">
<link rel="stylesheet" href="https://code.getmdl.io/1.2.1/material.blue-indigo.min.css">
<script defer src="https://code.getmdl.io/1.2.1/material.min.js"></script>
<style>
div.text-element {
text-align: center;
}
body {
background-color: #efefef;
}
</style>
</head>
<body>
{% if failure_zip_url is not none %}
<a href="{{ failure_zip_url }}">
<div class="mdl-color--primary" width="100%">
<h3>Download Image Zip</h3>
</div>
</a>
{% endif %}
{% for test_class, device_results in full_results.iteritems() %}
<div class="mdl-color--primary" width="100%">
<h3>{{ test_class }}</h3>
</div>
<div class="mdl-tabs mdl-js-tabs mdl-js-ripple-effect">
<div class="mdl-tabs__tab-bar">
{% for device_model, _ in device_results.iteritems() %}
<a href="#{{ device_model }}-panel" class="mdl-tabs__tab">{{ device_model }}</a>
{% endfor %}
</div>
{% for device_model, test_results in device_results.iteritems() %}
<div class="mdl-tabs__panel" id="{{ device_model }}-panel">
<div class="mdl-grid">
<div class="mdl-cell mdl-cell--3-col text-element"><b>Description</b></div>
<div class="mdl-cell mdl-cell--3-col text-element"><b>Golden</b></div>
<div class="mdl-cell mdl-cell--3-col text-element"><b>Failure</b></div>
{% if show_diffs %}
<div class="mdl-cell mdl-cell--3-col text-element"><b>Diff</b></div>
{% endif %}
</div>
{% for result in test_results %}
<div class="mdl-grid">
<div class="mdl-cell mdl-cell--3-col text-element">
{{ result['description'] }}
</div>
<div class="mdl-cell mdl-cell--3-col">
<a href="{{ result['golden_image'] }}">
<img class="mdl-shadow--2dp" src="{{ result['golden_image'] }}" width="100%">
</a>
</div>
<div class="mdl-cell mdl-cell--3-col mdl-shadow--2dp">
<a href="{{ result['failure_image'] }}">
<img src="{{ result['failure_image'] }}" width="100%">
</a>
</div>
{% if show_diffs %}
<div class="mdl-cell mdl-cell--3-col mdl-shadow--2dp">
<a href="{{ result['diff_image'] }}">
<img src="{{ result['diff_image'] }}" width="100%">
</a>
</div>
{% endif %}
</div>
{% endfor %}
</div>
{% endfor %}
</div>
{% endfor %}
</body>
</html>
build/android/test_runner.pydeps
View file @ 49c5485e
...@@ -114,6 +114,26 @@ ...@@ -114,6 +114,26 @@
../../third_party/catapult/tracing/tracing_build/__init__.py ../../third_party/catapult/tracing/tracing_build/__init__.py
../../third_party/catapult/tracing/tracing_build/trace2html.py ../../third_party/catapult/tracing/tracing_build/trace2html.py
../../third_party/catapult/tracing/tracing_project.py ../../third_party/catapult/tracing/tracing_project.py
../../third_party/jinja2/__init__.py
../../third_party/jinja2/_compat.py
../../third_party/jinja2/bccache.py
../../third_party/jinja2/compiler.py
../../third_party/jinja2/defaults.py
../../third_party/jinja2/environment.py
../../third_party/jinja2/exceptions.py
../../third_party/jinja2/filters.py
../../third_party/jinja2/lexer.py
../../third_party/jinja2/loaders.py
../../third_party/jinja2/nodes.py
../../third_party/jinja2/optimizer.py
../../third_party/jinja2/parser.py
../../third_party/jinja2/runtime.py
../../third_party/jinja2/tests.py
../../third_party/jinja2/utils.py
../../third_party/jinja2/visitor.py
../../third_party/markupsafe/__init__.py
../../third_party/markupsafe/_compat.py
../../third_party/markupsafe/_native.py
../../tools/swarming_client/libs/__init__.py ../../tools/swarming_client/libs/__init__.py
../../tools/swarming_client/libs/logdog/__init__.py ../../tools/swarming_client/libs/logdog/__init__.py
../../tools/swarming_client/libs/logdog/bootstrap.py ../../tools/swarming_client/libs/logdog/bootstrap.py
......
chrome/test/android/javatests/src/org/chromium/chrome/test/util/RenderUtils.java
View file @ 49c5485e
...@@ -7,6 +7,7 @@ package org.chromium.chrome.test.util; ...@@ -7,6 +7,7 @@ package org.chromium.chrome.test.util;
import android.app.Activity; import android.app.Activity;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.graphics.BitmapFactory; import android.graphics.BitmapFactory;
import android.graphics.Color;
import android.graphics.Point; import android.graphics.Point;
import android.os.Build; import android.os.Build;
import android.view.View; import android.view.View;
...@@ -28,8 +29,12 @@ import java.util.concurrent.Callable; ...@@ -28,8 +29,12 @@ import java.util.concurrent.Callable;
public class RenderUtils { public class RenderUtils {
private static final String TAG = "RenderUtils"; private static final String TAG = "RenderUtils";
private static final String DIFF_FOLDER_RELATIVE = "/diffs";
private static final String FAILURE_FOLDER_RELATIVE = "/failures"; private static final String FAILURE_FOLDER_RELATIVE = "/failures";
private static final String GOLDEN_FOLDER_RELATIVE = "/goldens";
/** /**
* This is a list of devices that we maintain golden images for. If render tests are being run * This is a list of devices that we maintain golden images for. If render tests are being run
* on a device in this list, golden images should exist and their absence is a test failure. * on a device in this list, golden images should exist and their absence is a test failure.
...@@ -45,6 +50,11 @@ public class RenderUtils { ...@@ -45,6 +50,11 @@ public class RenderUtils {
*/ */
private static final boolean REPORT_ONLY_DO_NOT_FAIL = true; private static final boolean REPORT_ONLY_DO_NOT_FAIL = true;
/**
* How many pixels can be different in an image before counting the images as different.
*/
private static final int PIXEL_DIFF_THRESHOLD = 0;
private enum ComparisonResult { MATCH, MISMATCH, GOLDEN_NOT_FOUND } private enum ComparisonResult { MATCH, MISMATCH, GOLDEN_NOT_FOUND }
/** /**
...@@ -72,11 +82,19 @@ public class RenderUtils { ...@@ -72,11 +82,19 @@ public class RenderUtils {
private final Activity mActivity; private final Activity mActivity;
private final String mGoldenFolder; private final String mGoldenFolder;
private final String mTestClass; private final String mTestClass;
private String mOutputDirectory;
public ViewRenderer(Activity activity, String goldenFolder, String testClass) { public ViewRenderer(Activity activity, String goldenFolder, String testClass) {
mActivity = activity; mActivity = activity;
mGoldenFolder = goldenFolder; mGoldenFolder = goldenFolder;
mTestClass = testClass; mTestClass = testClass;
// Render test will output results to subdirectory of |goldenFolder| unless
// --render-test-output-dir is passed.
mOutputDirectory = CommandLine.getInstance().getSwitchValue("render-test-output-dir");
if (mOutputDirectory == null) {
mOutputDirectory = UrlUtils.getIsolatedTestFilePath(mGoldenFolder);
}
} }
/** /**
...@@ -93,20 +111,34 @@ public class RenderUtils { ...@@ -93,20 +111,34 @@ public class RenderUtils {
*/ */
public void renderAndCompare(final View view, String id) throws IOException { public void renderAndCompare(final View view, String id) throws IOException {
// Compare the image against the Golden. // Compare the image against the Golden.
Bitmap bitmap = ThreadUtils.runOnUiThreadBlockingNoException(new Callable<Bitmap>() { Bitmap testBitmap =
ThreadUtils.runOnUiThreadBlockingNoException(new Callable<Bitmap>() {
@Override @Override
public Bitmap call() throws Exception { public Bitmap call() throws Exception {
return UiUtils.generateScaledScreenshot(view, 0, Bitmap.Config.ARGB_8888); return UiUtils.generateScaledScreenshot(
view, 0, Bitmap.Config.ARGB_8888);
} }
}); });
String imagename = imageName(mActivity, mTestClass, id); String imagename = imageName(mActivity, mTestClass, id);
File goldenFile = createPath(mGoldenFolder, imagename);
ComparisonResult result = compareBmpToGolden(bitmap, goldenFile);
if (REPORT_ONLY_DO_NOT_FAIL && !isGenerateMode()) { File goldenFile =
Log.d(TAG, "Image comparison for %s: %s", id, result.name()); createPath(UrlUtils.getIsolatedTestFilePath(mGoldenFolder), imagename);
return; BitmapFactory.Options options = new BitmapFactory.Options();
options.inPreferredConfig = testBitmap.getConfig();
Bitmap goldenBitmap = BitmapFactory.decodeFile(goldenFile.getAbsolutePath(), options);
Bitmap diffBitmap = null;
ComparisonResult result = null;
if (goldenBitmap == null) {
result = ComparisonResult.GOLDEN_NOT_FOUND;
} else {
diffBitmap = Bitmap.createBitmap(
Math.max(testBitmap.getWidth(), goldenBitmap.getWidth()),
Math.max(testBitmap.getHeight(), goldenBitmap.getHeight()),
testBitmap.getConfig());
result = compareBitmapToGolden(testBitmap, goldenBitmap, diffBitmap);
} }
if (result == ComparisonResult.MATCH) { if (result == ComparisonResult.MATCH) {
...@@ -116,12 +148,28 @@ public class RenderUtils { ...@@ -116,12 +148,28 @@ public class RenderUtils {
return; return;
} }
// Save the rendered View. File failureOutputFile =
File failureFile = createPath(mGoldenFolder + FAILURE_FOLDER_RELATIVE, imagename); createPath(mOutputDirectory + FAILURE_FOLDER_RELATIVE, imagename);
saveBitmap(bitmap, failureFile); saveBitmap(testBitmap, failureOutputFile);
if (result != ComparisonResult.GOLDEN_NOT_FOUND) {
File goldenOutputFile =
createPath(mOutputDirectory + GOLDEN_FOLDER_RELATIVE, imagename);
saveBitmap(goldenBitmap, goldenOutputFile);
File diffOutputFile =
createPath(mOutputDirectory + DIFF_FOLDER_RELATIVE, imagename);
saveBitmap(diffBitmap, diffOutputFile);
}
if (isGenerateMode()) { if (isGenerateMode()) {
Log.i(TAG, "%s - generated image saved to %s.", id, failureFile.getAbsolutePath()); Log.i(TAG, "%s - generated image saved to %s.", id,
failureOutputFile.getAbsolutePath());
return;
}
if (REPORT_ONLY_DO_NOT_FAIL) {
Log.d(TAG, "Image comparison for %s: %s", id, result.name());
return; return;
} }
...@@ -137,8 +185,7 @@ public class RenderUtils { ...@@ -137,8 +185,7 @@ public class RenderUtils {
} else if (result == ComparisonResult.MISMATCH) { } else if (result == ComparisonResult.MISMATCH) {
throw new ImageMismatchException( throw new ImageMismatchException(
String.format("Image comparison failed on %s. Failure image saved to %s.", String.format("Image comparison failed on %s. Failure image saved to %s.",
id, failureFile.getAbsolutePath())); id, failureOutputFile.getAbsolutePath()));
} }
} }
} }
...@@ -163,6 +210,9 @@ public class RenderUtils { ...@@ -163,6 +210,9 @@ public class RenderUtils {
/** /**
* Creates an image name combining the image description with details about the device * Creates an image name combining the image description with details about the device
* (eg model, current orientation). * (eg model, current orientation).
*
* This function must be kept in sync with |RE_RENDER_IMAGE_NAME| from
* src/build/android/pylib/local/device/local_device_instrumentation_test_run.py.
*/ */
private static String imageName(Activity activity, String testClass, String desc) { private static String imageName(Activity activity, String testClass, String desc) {
Point outSize = new Point(); Point outSize = new Point();
...@@ -185,14 +235,13 @@ public class RenderUtils { ...@@ -185,14 +235,13 @@ public class RenderUtils {
} }
/** /**
* Convenience method to create a File pointing to |filename| in |folder| in the external * Convenience method to create a File pointing to |filename| in |folder|.
* storage directory.
* @throws IOException * @throws IOException
*/ */
private static File createPath(String folder, String filename) throws IOException { private static File createPath(String folder, String filename) throws IOException {
File path = new File(UrlUtils.getIsolatedTestFilePath(folder)); File path = new File(folder);
if (!path.exists()) { if (!path.exists()) {
if (!path.mkdir()) { if (!path.mkdirs()) {
throw new IOException("Could not create " + path.getAbsolutePath()); throw new IOException("Could not create " + path.getAbsolutePath());
} }
} }
...@@ -202,12 +251,111 @@ public class RenderUtils { ...@@ -202,12 +251,111 @@ public class RenderUtils {
/** /**
* Returns whether the given |bitmap| is equal to the one stored in |file|. * Returns whether the given |bitmap| is equal to the one stored in |file|.
*/ */
private static ComparisonResult compareBmpToGolden(Bitmap bitmap, File file) { private static ComparisonResult compareBitmapToGolden(Bitmap test, Bitmap golden, Bitmap diff) {
BitmapFactory.Options options = new BitmapFactory.Options(); int maxWidth = Math.max(test.getWidth(), golden.getWidth());
options.inPreferredConfig = bitmap.getConfig(); int maxHeight = Math.max(test.getHeight(), golden.getHeight());
Bitmap golden = BitmapFactory.decodeFile(file.getAbsolutePath(), options); int minWidth = Math.min(test.getWidth(), golden.getWidth());
int minHeight = Math.min(test.getHeight(), golden.getHeight());
int diffPixelsCount = comparePixels(test, golden, diff, 0, 0, minWidth, 0, minHeight)
+ compareSizes(diff, minWidth, maxWidth, minHeight, maxHeight);
if (golden == null) return ComparisonResult.GOLDEN_NOT_FOUND; if (diffPixelsCount > PIXEL_DIFF_THRESHOLD) {
return bitmap.sameAs(golden) ? ComparisonResult.MATCH : ComparisonResult.MISMATCH; return ComparisonResult.MISMATCH;
}
return ComparisonResult.MATCH;
}
/**
* Compares two bitmaps pixel-wise.
*
* @param testImage Bitmap of test image.
*
* @param goldenImage Bitmap of golden image.
*
* @param diffImage This is an output argument. Function will set pixels in the |diffImage| to
* either transparent or red depending on whether that pixel differed in the golden and test
* bitmaps. diffImage should have its width and height be the max width and height of the
* golden and test bitmaps.
*
* @param diffThreshold Threshold for when to consider two color values as different. These
* values are 8 bit (256) so this threshold value should be in range 0-256.
*
* @param startWidth Start x-coord to start diffing the Bitmaps.
*
* @param endWidth End x-coord to start diffing the Bitmaps.
*
* @param startHeight Start y-coord to start diffing the Bitmaps.
*
* @param endHeight End x-coord to start diffing the Bitmaps.
*
* @return Returns number of pixels that differ between |goldenImage| and |testImage|
*/
private static int comparePixels(Bitmap testImage, Bitmap goldenImage, Bitmap diffImage,
int diffThreshold, int startWidth, int endWidth, int startHeight, int endHeight) {
int diffPixels = 0;
for (int x = startWidth; x < endWidth; x++) {
for (int y = startHeight; y < endHeight; y++) {
int goldenImageColor = goldenImage.getPixel(x, y);
int testImageColor = testImage.getPixel(x, y);
int redDiff = Math.abs(Color.red(goldenImageColor) - Color.red(testImageColor));
int blueDiff =
Math.abs(Color.green(goldenImageColor) - Color.green(testImageColor));
int greenDiff = Math.abs(Color.blue(goldenImageColor) - Color.blue(testImageColor));
int alphaDiff =
Math.abs(Color.alpha(goldenImageColor) - Color.alpha(testImageColor));
if (redDiff > diffThreshold || blueDiff > diffThreshold || greenDiff > diffThreshold
|| alphaDiff > diffThreshold) {
diffPixels++;
diffImage.setPixel(x, y, Color.RED);
} else {
diffImage.setPixel(x, y, Color.TRANSPARENT);
}
}
}
return diffPixels;
}
/**
* Compares two bitmaps size.
*
* @param diffImage This is an output argument. Function will set pixels in the |diffImage| to
* either transparent or red depending on whether that pixel coordinate occurs in the
* dimensions of the golden and not the test bitmap or vice-versa.
*
* @param minWidth Min width of golden and test bitmaps.
*
* @param maxWidth Max width of golden and test bitmaps.
*
* @param minHeight Min height of golden and test bitmaps.
*
* @param maxHeight Max height of golden and test bitmaps.
*
* @return Returns number of pixels that differ between |goldenImage| and |testImage| due to
* their size.
*/
private static int compareSizes(
Bitmap diffImage, int minWidth, int maxWidth, int minHeight, int maxHeight) {
int diffPixels = 0;
if (maxWidth > minWidth) {
for (int x = minWidth; x < maxWidth; x++) {
for (int y = 0; y < maxHeight; y++) {
diffImage.setPixel(x, y, Color.RED);
}
}
diffPixels += (maxWidth - minWidth) * maxHeight;
}
if (maxHeight > minHeight) {
for (int x = 0; x < maxWidth; x++) {
for (int y = minHeight; y < maxHeight; y++) {
diffImage.setPixel(x, y, Color.RED);
}
}
diffPixels += (maxHeight - minHeight) * minWidth;
}
return diffPixels;
} }
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment