[I-Spy] Add support for rebaselining expectations from the web UI.

BUG=318865
TEST=unittests included
NOTRY=True

Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=240226

Review URL: https://codereview.chromium.org/106523003

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@243140 0039d316-1c4b-4281-b951-d872f2087c98

chrome/test/functional/ispy/client/chrome_utils_unittest.py

-#!/usr/bin/env python
-#
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import json
-import unittest
-from PIL import Image
-
-import chrome_utils
-from ..common import mock_cloud_bucket
-
-
-class ChromeUtilsTest(unittest.TestCase):
-  """Unittest for ChromeUtils."""
-
-  def setUp(self):
-    self.cloud_bucket = mock_cloud_bucket.MockCloudBucket()
-    self.white_utils = chrome_utils.ChromeUtils(
-        self.cloud_bucket, 'versions.json',
-        lambda: Image.new('RGBA', (10, 10), (255, 255, 255, 255)))
-    self.black_utils = chrome_utils.ChromeUtils(
-        self.cloud_bucket, 'versions.json',
-        lambda: Image.new('RGBA', (10, 10), (0, 0, 0, 255)))
-
-  def testGenerateExpectationsRunComparison(self):
-    self.white_utils.GenerateExpectation('device', 'test', '1.1.1.1')
-    self.white_utils.UpdateExpectationVersion('1.1.1.1')
-    self.white_utils.PerformComparison('test1', 'device', 'test', '1.1.1.1')
-    expect_name = self.white_utils._CreateExpectationName(
-        'device', 'test', '1.1.1.1')
-    self.assertFalse(self.white_utils._ispy.FailureExists('test1', expect_name))
-    self.black_utils.PerformComparison('test2', 'device', 'test', '1.1.1.1')
-    self.assertTrue(self.white_utils._ispy.FailureExists('test2', expect_name))
-
-  def testUpdateExpectationVersion(self):
-    self.white_utils.UpdateExpectationVersion('1.0.0.0')
-    self.white_utils.UpdateExpectationVersion('1.0.4.0')
-    self.white_utils.UpdateExpectationVersion('2.1.5.0')
-    self.white_utils.UpdateExpectationVersion('1.1.5.0')
-    self.white_utils.UpdateExpectationVersion('0.0.0.0')
-    self.white_utils.UpdateExpectationVersion('1.1.5.0')
-    self.white_utils.UpdateExpectationVersion('0.0.0.1')
-    versions = json.loads(self.cloud_bucket.DownloadFile('versions.json'))
-    self.assertEqual(versions,
-        ['2.1.5.0', '1.1.5.0', '1.0.4.0', '1.0.0.0', '0.0.0.1', '0.0.0.0'])
-
-
-if __name__ == '__main__':
-  unittest.main()

chrome/test/functional/ispy/client/dom.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+def GetScriptToWaitForUnchangingDOM():
+  """Gets Javascript that waits until the DOM is stable for 5 seconds.
+
+  Times out if the DOM is not stable within 30 seconds.
+
+  Returns:
+    Javascript as a string.
+  """
+  return """
+    var target = document.body;
+    var callback = arguments[arguments.length - 1]
+
+    var timeout_id = setTimeout(function() {
+      callback()
+    }, 5000);
+
+    var observer = new MutationObserver(function(mutations) {
+      clearTimeout(timeout_id);
+      timeout_id = setTimeout(function() {
+        callback();
+      }, 5000);
+    }).observe(target, {attributes: true, childList: true,
+      characterData: true, subtree: true});
+  """

chrome/test/functional/ispy/client/chrome_utils.py → chrome/test/functional/ispy/common/chrome_utils.py

-# Copyright 2013 The Chromium Authors. All rights reserved.
+# Copyright 2014 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import json
 import logging
 import os
-import time
 from distutils.version import LooseVersion
 from PIL import Image
 
-from ..common import cloud_bucket
-from ..common import ispy_utils
+import cloud_bucket
+import ispy_utils
 
 
 class ChromeUtils(object):
   """A utility for using ISpy with Chrome."""
 
-  def __init__(self, cloud_bucket, version_file, screenshot_func):
+  def __init__(self, cloud_bucket):
     """Initializes the utility class.
 
     Args:
       cloud_bucket: a BaseCloudBucket in which to the version file,
           expectations and results are to be stored.
-      version_file: path to the version file in the cloud bucket. The version
-          file contains a json list of ordered Chrome versions for which
-          expectations exist.
-      screenshot_func: a function that returns a PIL.Image.
     """
     self._cloud_bucket = cloud_bucket
-    self._version_file = version_file
-    self._screenshot_func = screenshot_func
     self._ispy = ispy_utils.ISpyUtils(self._cloud_bucket)
-    with open(
-        os.path.join(os.path.dirname(__file__), 'wait_on_ajax.js'), 'r') as f:
-      self._wait_for_unchanging_dom_script = f.read()
+    self._rebaselineable_cache = {}
 
-  def UpdateExpectationVersion(self, chrome_version):
+  def UpdateExpectationVersion(self, chrome_version, version_file):
     """Updates the most recent expectation version to the Chrome version.
 
     Should be called after generating a new set of expectations.
 
     Args:
       chrome_version: the chrome version as a string of the form "31.0.123.4".
+      version_file: path to the version file in the cloud bucket. The version
+          file contains a json list of ordered Chrome versions for which
+          expectations exist.
     """
     insert_pos = 0
     expectation_versions = []
     try:
-      expectation_versions = self._GetExpectationVersionList()
+      expectation_versions = self._GetExpectationVersionList(version_file)
       if expectation_versions:
         try:
           version = self._GetExpectationVersion(
@@ -61,7 +55,7 @@ class ChromeUtils(object):
     expectation_versions.insert(insert_pos, chrome_version)
     logging.info('Updating expectation version...')
     self._cloud_bucket.UploadFile(
-        self._version_file, json.dumps(expectation_versions),
+        version_file, json.dumps(expectation_versions),
         'application/json')
 
   def _GetExpectationVersion(self, chrome_version, expectation_versions):
@@ -81,12 +75,21 @@ class ChromeUtils(object):
         return version
     raise Exception('No expectation exists for Chrome %s' % chrome_version)
 
-  def _GetExpectationVersionList(self):
-    """Gets the list of expectation versions from google storage."""
-    return json.loads(self._cloud_bucket.DownloadFile(self._version_file))
+  def _GetExpectationVersionList(self, version_file):
+    """Gets the list of expectation versions from google storage.
+
+    Args:
+      version_file: path to the version file in the cloud bucket. The version
+          file contains a json list of ordered Chrome versions for which
+          expectations exist.
+
+    Returns:
+      Ordered list of Chrome versions.
+    """
+    return json.loads(self._cloud_bucket.DownloadFile(version_file))
 
   def _GetExpectationNameWithVersion(self, device_type, expectation,
-                                     chrome_version):
+                                     chrome_version, version_file):
     """Get the expectation to be used with the current Chrome version.
 
     Args:
@@ -98,7 +101,7 @@ class ChromeUtils(object):
       Version as an integer.
     """
     version = self._GetExpectationVersion(
-        chrome_version, self._GetExpectationVersionList())
+        chrome_version, self._GetExpectationVersionList(version_file))
     return self._CreateExpectationName(device_type, expectation, version)
 
   def _CreateExpectationName(self, device_type, expectation, version):
@@ -114,47 +117,110 @@ class ChromeUtils(object):
     """
     return '%s:%s(%s)' % (device_type, expectation, version)
 
-  def GenerateExpectation(self, device_type, expectation, chrome_version):
-    """Take screenshots and store as an expectation in I-Spy.
+  def GenerateExpectation(self, device_type, expectation, chrome_version,
+                          version_file, screenshots):
+    """Create an expectation for I-Spy.
 
     Args:
       device_type: string identifier for the device type.
       expectation: name for the expectation to generate.
       chrome_version: the chrome version as a string of the form "31.0.123.4".
+      screenshots: a list of similar PIL.Images.
     """
     # https://code.google.com/p/chromedriver/issues/detail?id=463
-    time.sleep(1)
     expectation_with_version = self._CreateExpectationName(
         device_type, expectation, chrome_version)
     if self._ispy.ExpectationExists(expectation_with_version):
       logging.warning(
           'I-Spy expectation \'%s\' already exists, overwriting.',
           expectation_with_version)
-    screenshots = [self._screenshot_func() for _ in range(8)]
     logging.info('Generating I-Spy expectation...')
     self._ispy.GenerateExpectation(expectation_with_version, screenshots)
 
   def PerformComparison(self, test_run, device_type, expectation,
-                        chrome_version):
-    """Take a screenshot and compare it with the given expectation in I-Spy.
+                        chrome_version, version_file, screenshot):
+    """Compare a screenshot with the given expectation in I-Spy.
 
     Args:
       test_run: name for the test run.
       device_type: string identifier for the device type.
       expectation: name for the expectation to compare against.
       chrome_version: the chrome version as a string of the form "31.0.123.4".
+      screenshot: a PIL.Image to compare.
     """
     # https://code.google.com/p/chromedriver/issues/detail?id=463
-    time.sleep(1)
-    screenshot = self._screenshot_func()
     logging.info('Performing I-Spy comparison...')
     self._ispy.PerformComparison(
         test_run,
         self._GetExpectationNameWithVersion(
-            device_type, expectation, chrome_version),
+            device_type, expectation, chrome_version, version_file),
         screenshot)
 
-  def GetScriptToWaitForUnchangingDOM(self):
-    """Returns a JavaScript script that waits for the DOM to stop changing."""
-    return self._wait_for_unchanging_dom_script
+  def CanRebaselineToTestRun(self, test_run):
+    """Returns whether the test run has associated expectations.
+
+    Returns:
+      True if RebaselineToTestRun() can be called for this test run.
+    """
+    if test_run in self._rebaselineable_cache:
+      return True
+    return self._cloud_bucket.FileExists(
+        ispy_utils.GetTestRunPath(test_run, 'rebaseline.txt'))
+
+  def RebaselineToTestRun(self, test_run):
+    """Update the version file to use expectations associated with |test_run|.
+
+    Args:
+      test_run: The name of the test run to rebaseline.
+    """
+    rebaseline_path = ispy_utils.GetTestRunPath(test_run, 'rebaseline.txt')
+    rebaseline_attrib = json.loads(
+        self._cloud_bucket.DownloadFile(rebaseline_path))
+    self.UpdateExpectationVersion(
+        rebaseline_attrib['version'], rebaseline_attrib['version_file'])
+    self._cloud_bucket.RemoveFile(rebaseline_path)
+
+  def _SetTestRunRebaselineable(self, test_run, chrome_version, version_file):
+    """Writes a JSON file containing the data needed to rebaseline.
+
+    Args:
+      test_run: The name of the test run to add the rebaseline file to.
+      chrome_version: the chrome version that can be rebaselined to (must have
+        associated Expectations).
+      version_file: the path of the version file associated with the test run.
+    """
+    self._rebaselineable_cache[test_run] = True
+    self._cloud_bucket.UploadFile(
+        ispy_utils.GetTestRunPath(test_run, 'rebaseline.txt'),
+        json.dumps({
+            'version': chrome_version,
+            'version_file': version_file}),
+        'application/json')
+
+  def PerformComparisonAndPrepareExpectation(self, test_run, device_type,
+                                             expectation, chrome_version,
+                                             version_file, screenshots):
+    """Perform comparison and generate an expectation that can used later.
+
+    The test run web UI will have a button to set the Expectations generated for
+    this version as the expectation for comparison with later versions.
+
+    Args:
+      test_run: The name of the test run to add the rebaseline file to.
+      device_type: string identifier for the device type.
+      chrome_version: the chrome version that can be rebaselined to (must have
+        associated Expectations).
+      version_file: the path of the version file associated with the test run.
+      screenshot: a list of similar PIL.Images.
+    """
+    if not self.CanRebaselineToTestRun(test_run):
+      self._SetTestRunRebaselineable(test_run, chrome_version, version_file)
+    expectation_with_version = self._CreateExpectationName(
+        device_type, expectation, chrome_version)
+    self._ispy.GenerateExpectation(expectation_with_version, screenshots)
+    self._ispy.PerformComparison(
+        test_run,
+        self._GetExpectationNameWithVersion(
+            device_type, expectation, chrome_version, version_file),
+        screenshots[-1])
 

chrome/test/functional/ispy/common/chrome_utils_unittest.py

+#!/usr/bin/env python
+#
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import unittest
+from PIL import Image
+
+import chrome_utils
+from ..common import cloud_bucket
+from ..common import mock_cloud_bucket
+
+
+class ChromeUtilsTest(unittest.TestCase):
+  """Unittest for ChromeUtils."""
+
+  def setUp(self):
+    self.cloud_bucket = mock_cloud_bucket.MockCloudBucket()
+    self.chrome_util = chrome_utils.ChromeUtils(self.cloud_bucket)
+    self.white_img = Image.new('RGBA', (10, 10), (255, 255, 255, 255))
+    self.black_img = Image.new('RGBA', (10, 10), (0, 0, 0, 255))
+
+  def testGenerateExpectationsRunComparison(self):
+    self.chrome_util.GenerateExpectation(
+        'device', 'test', '1.1.1.1', 'versions.json',
+        [self.white_img, self.white_img])
+    self.chrome_util.UpdateExpectationVersion('1.1.1.1', 'versions.json')
+    self.chrome_util.PerformComparison(
+        'test1', 'device', 'test', '1.1.1.1', 'versions.json', self.white_img)
+    expect_name = self.chrome_util._CreateExpectationName(
+        'device', 'test', '1.1.1.1')
+    self.assertFalse(self.chrome_util._ispy.FailureExists('test1', expect_name))
+    self.chrome_util.PerformComparison(
+        'test2', 'device', 'test', '1.1.1.1','versions.json', self.black_img)
+    self.assertTrue(self.chrome_util._ispy.FailureExists('test2', expect_name))
+
+  def testUpdateExpectationVersion(self):
+    self.chrome_util.UpdateExpectationVersion('1.0.0.0', 'versions.json')
+    self.chrome_util.UpdateExpectationVersion('1.0.4.0', 'versions.json')
+    self.chrome_util.UpdateExpectationVersion('2.1.5.0', 'versions.json')
+    self.chrome_util.UpdateExpectationVersion('1.1.5.0', 'versions.json')
+    self.chrome_util.UpdateExpectationVersion('0.0.0.0', 'versions.json')
+    self.chrome_util.UpdateExpectationVersion('1.1.5.0', 'versions.json')
+    self.chrome_util.UpdateExpectationVersion('0.0.0.1', 'versions.json')
+    versions = json.loads(self.cloud_bucket.DownloadFile('versions.json'))
+    self.assertEqual(versions,
+        ['2.1.5.0', '1.1.5.0', '1.0.4.0', '1.0.0.0', '0.0.0.1', '0.0.0.0'])
+
+  def testPerformComparisonAndPrepareExpectation(self):
+    self.assertFalse(self.chrome_util.CanRebaselineToTestRun('test'))
+    self.assertRaises(
+        cloud_bucket.FileNotFoundError,
+        self.chrome_util.PerformComparisonAndPrepareExpectation,
+        'test', 'device', 'expect', '1.0', 'versions.json',
+        [self.white_img, self.white_img])
+    self.assertTrue(self.chrome_util.CanRebaselineToTestRun('test'))
+    self.chrome_util.RebaselineToTestRun('test')
+    versions = json.loads(self.cloud_bucket.DownloadFile('versions.json'))
+    self.assertEqual(versions, ['1.0'])
+    self.chrome_util.PerformComparisonAndPrepareExpectation(
+        'test1', 'device', 'expect', '1.1', 'versions.json',
+        [self.white_img, self.white_img])
+
+
+
+if __name__ == '__main__':
+  unittest.main()

chrome/test/functional/ispy/common/ispy_utils.py

@@ -2,7 +2,10 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-"""Utilities for managing I-Spy test results in Google Cloud Storage."""
+"""Internal utilities for managing I-Spy test results in Google Cloud Storage.
+
+See the ispy.client.chrome_utils module for the external API.
+"""
 
 import collections
 import itertools
@@ -44,7 +47,20 @@ def GetFailurePath(test_run, expectation, file_name=''):
   Returns:
     the path as a string relative to the bucket.
   """
-  return 'failures/%s/%s/%s' % (test_run, expectation, file_name)
+  return GetTestRunPath(test_run, '%s/%s' % (expectation, file_name))
+
+
+def GetTestRunPath(test_run, file_name=''):
+  """Get the path to a the given test run.
+
+  Args:
+    test_run: name of the test run.
+    file_name: name of the file.
+
+  Returns:
+    the path as a string relative to the bucket.
+  """
+  return 'failures/%s/%s' % (test_run, file_name)
 
 
 class ISpyUtils(object):
@@ -92,7 +108,6 @@ class ISpyUtils(object):
     """
     self.cloud_bucket.UpdateFile(full_path, image_tools.EncodePNG(image))
 
-
   def GenerateExpectation(self, expectation, images):
     """Creates and uploads an expectation to GS from a set of images and name.
 

chrome/test/functional/ispy/server/app.py

@@ -2,16 +2,21 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+import os
+import sys
 import webapp2
 
+sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
 import debug_view_handler
 import image_handler
 import main_view_handler
+import rebaseline_handler
 import update_mask_handler
 
 
 application = webapp2.WSGIApplication(
     [('/update_mask', update_mask_handler.UpdateMaskHandler),
+     ('/rebaseline', rebaseline_handler.RebaselineHandler),
      ('/debug_view', debug_view_handler.DebugViewHandler),
      ('/image', image_handler.ImageHandler),
      ('/', main_view_handler.MainViewHandler)], debug=True)

chrome/test/functional/ispy/server/debug_view_handler.py

@@ -9,7 +9,7 @@ import os
 import sys
 import webapp2
 
-from ..common import ispy_utils
+from common import ispy_utils
 
 import views
 

chrome/test/functional/ispy/server/gs_bucket.py

@@ -8,7 +8,7 @@ import sys
 
 import cloudstorage
 
-from ..common import cloud_bucket
+from common import cloud_bucket
 
 
 class GoogleCloudStorageBucket(cloud_bucket.BaseCloudBucket):

chrome/test/functional/ispy/server/image_handler.py

@@ -9,8 +9,8 @@ import os
 import sys
 import webapp2
 
-from ..common import cloud_bucket
-from ..common import constants
+from common import cloud_bucket
+from common import constants
 
 import gs_bucket
 

chrome/test/functional/ispy/server/main_view_handler.py

@@ -11,8 +11,9 @@ import re
 import sys
 import webapp2
 
-from ..common import constants
-from ..common import ispy_utils
+from common import chrome_utils
+from common import constants
+from common import ispy_utils
 
 import gs_bucket
 import views
@@ -67,19 +68,19 @@ class MainViewHandler(webapp2.RequestHandler):
     """
     paths = set([path for path in ispy.GetAllPaths('failures/' + test_run)
                  if path.endswith('actual.png')])
+    can_rebaseline = chrome_utils.ChromeUtils(
+        ispy.cloud_bucket).CanRebaselineToTestRun(test_run)
     rows = [self._CreateRow(test_run, path, ispy) for path in paths]
-    if rows:
-      # Function that sorts by the different_pixels field in the failure-info.
-      def _Sorter(a, b):
-        return cmp(b['percent_different'],
-                   a['percent_different'])
-      template = JINJA.get_template('main_view.html')
-      self.response.write(
-          template.render({'comparisons': sorted(rows, _Sorter),
-                           'test_run': test_run}))
-    else:
-      template = JINJA.get_template('empty_view.html')
-      self.response.write(template.render())
+
+    # Function that sorts by the different_pixels field in the failure-info.
+    def _Sorter(a, b):
+      return cmp(b['percent_different'],
+                 a['percent_different'])
+    template = JINJA.get_template('main_view.html')
+    self.response.write(
+        template.render({'comparisons': sorted(rows, _Sorter),
+                         'test_run': test_run,
+                         'can_rebaseline': can_rebaseline}))
 
   def _CreateRow(self, test_run, path, ispy):
     """Creates one failure-row.

chrome/test/functional/ispy/server/rebaseline_handler.py

+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Request Handler that updates the Expectation version."""
+
+import webapp2
+
+from common import constants
+from common import chrome_utils
+
+import gs_bucket
+
+
+class RebaselineHandler(webapp2.RequestHandler):
+  """Request handler to allow test mask updates."""
+
+  def post(self):
+    """Accepts post requests.
+
+    Expects a test_run as a parameter and updates the associated version file to
+    use the expectations associated with that test run.
+    """
+    test_run = self.request.get('test_run')
+
+    # Fail if test_run parameter is missing.
+    if not test_run:
+      self.response.header['Content-Type'] = 'json/application'
+      self.response.write(json.dumps(
+          {'error': '\'test_run\' must be supplied to rebaseline.'}))
+      return
+    # Otherwise, set up the utilities.
+    bucket = gs_bucket.GoogleCloudStorageBucket(constants.BUCKET)
+    chrome_util = chrome_utils.ChromeUtils(bucket)
+    # Update versions file.
+    try:
+      chrome_util.RebaselineToTestRun(test_run)
+    except:
+      self.response.header['Content-Type'] = 'json/application'
+      self.response.write(json.dumps(
+        {'error': 'Can not rebaseline to the given test run.'}))
+      return
+    # Redirect back to the sites list for the test run.
+    self.redirect('/?test_run=%s' % test_run)

chrome/test/functional/ispy/server/update_mask_handler.py

@@ -9,9 +9,9 @@ import re
 import sys
 import os
 
-from ..common import constants
-from ..common import image_tools
-from ..common import ispy_utils
+from common import constants
+from common import image_tools
+from common import ispy_utils
 
 import gs_bucket
 

chrome/test/functional/ispy/server/views/empty_view.html

-<!DOCTYPE html>
-<html>
-  <head>
-    <title>No Failures</title>
-  </head>
-  <body>
-    <h2>No Failures.</h2>
-  </body>
-</html>

chrome/test/functional/ispy/server/views/main_view.html

@@ -24,8 +24,25 @@
         height: 350px;
       }
     </style>
+
+    <script language="javascript">
+    var confirmSubmit = function() {
+      return confirm("The screenshots generated with this version of chrome will be used as the expected images for future comparisions. Are you sure?");
+    }
+    </script>
   </head>
   <body>
+    <h3>Test Run: {{ test_run }}</h3>
+    {% if can_rebaseline and comparisons %}
+      <form action="/rebaseline" method="post" onsubmit="return confirmSubmit();">
+        <input type="hidden" name="test_run" value="{{ test_run }}"/>
+        <input type="submit" value="This Chrome version looks good!"/>
+      </form>
+      <br>
+    {% endif %}
+    {% if not comparisons %}
+      <h2>No failures.</h2>
+    {% endif %}
     {% for comp in comparisons %}
       <div class="row">
         <div class="cell">