[I-Spy] Add utilities to help manage expectations that track Chrome versions.

BUG=297210
TEST=included
NOTRY=True

Review URL: https://codereview.chromium.org/58623002

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@233436 0039d316-1c4b-4281-b951-d872f2087c98

chrome/test/functional/ispy/client/chrome_utils.py

+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import logging
+import os
+import time
+from distutils.version import LooseVersion
+from PIL import Image
+
+from ..common import cloud_bucket
+from ..common import ispy_utils
+
+
+class ChromeUtils(object):
+  """A utility for using ISpy with Chrome."""
+
+  def __init__(self, cloud_bucket, version_file, screenshot_func):
+    """Initializes the utility class.
+
+    Args:
+      cloud_bucket: a BaseCloudBucket in which to the version file,
+          expectations and results are to be stored.
+      version_file: path to the version file in the cloud bucket. The version
+          file contains a json list of ordered Chrome versions for which
+          expectations exist.
+      screenshot_func: a function that returns a PIL.Image.
+    """
+    self._cloud_bucket = cloud_bucket
+    self._version_file = version_file
+    self._screenshot_func = screenshot_func
+    self._ispy = ispy_utils.ISpyUtils(self._cloud_bucket)
+    with open(
+        os.path.join(os.path.dirname(__file__), 'wait_on_ajax.js'), 'r') as f:
+      self._wait_for_unchanging_dom_script = f.read()
+
+  def UpdateExpectationVersion(self, chrome_version):
+    """Updates the most recent expectation version to the Chrome version.
+
+    Should be called after generating a new set of expectations.
+
+    Args:
+      chrome_version: the chrome version as a string of the form "31.0.123.4".
+    """
+    insert_pos = 0
+    expectation_versions = []
+    try:
+      expectation_versions = self._GetExpectationVersionList()
+      if expectation_versions:
+        try:
+          version = self._GetExpectationVersion(
+              chrome_version, expectation_versions)
+          if version == chrome_version:
+            return
+          insert_pos = expectation_versions.index(version)
+        except:
+          insert_pos = len(expectation_versions)
+    except cloud_bucket.FileNotFoundError:
+      pass
+    expectation_versions.insert(insert_pos, chrome_version)
+    logging.info('Updating expectation version...')
+    self._cloud_bucket.UploadFile(
+        self._version_file, json.dumps(expectation_versions),
+        'application/json')
+
+  def _GetExpectationVersion(self, chrome_version, expectation_versions):
+    """Returns the expectation version for the given Chrome version.
+
+    Args:
+      chrome_version: the chrome version as a string of the form "31.0.123.4".
+      expectation_versions: Ordered list of Chrome versions for which
+        expectations exist, as stored in the version file.
+
+    Returns:
+      Expectation version string.
+    """
+    # Find the closest version that is not greater than the chrome version.
+    for version in expectation_versions:
+      if LooseVersion(version) <= LooseVersion(chrome_version):
+        return version
+    raise Exception('No expectation exists for Chrome %s' % chrome_version)
+
+  def _GetExpectationVersionList(self):
+    """Gets the list of expectation versions from google storage."""
+    return json.loads(self._cloud_bucket.DownloadFile(self._version_file))
+
+  def _GetExpectationNameWithVersion(self, device_type, expectation,
+                                     chrome_version):
+    """Get the expectation to be used with the current Chrome version.
+
+    Args:
+      device_type: string identifier for the device type.
+      expectation: name for the expectation to generate.
+      chrome_version: the chrome version as a string of the form "31.0.123.4".
+
+    Returns:
+      Version as an integer.
+    """
+    version = self._GetExpectationVersion(
+        chrome_version, self._GetExpectationVersionList())
+    return self._CreateExpectationName(device_type, expectation, version)
+
+  def _CreateExpectationName(self, device_type, expectation, version):
+    """Create the full expectation name from the expectation and version.
+
+    Args:
+      device_type: string identifier for the device type, example: mako
+      expectation: base name for the expectation, example: google.com
+      version: expectation version, example: 31.0.23.1
+
+    Returns:
+      Full expectation name as a string, example: mako:google.com(31.0.23.1)
+    """
+    return '%s:%s(%s)' % (device_type, expectation, version)
+
+  def GenerateExpectation(self, device_type, expectation, chrome_version):
+    """Take screenshots and store as an expectation in I-Spy.
+
+    Args:
+      device_type: string identifier for the device type.
+      expectation: name for the expectation to generate.
+      chrome_version: the chrome version as a string of the form "31.0.123.4".
+    """
+    # https://code.google.com/p/chromedriver/issues/detail?id=463
+    time.sleep(1)
+    expectation_with_version = self._CreateExpectationName(
+        device_type, expectation, chrome_version)
+    if self._ispy.ExpectationExists(expectation_with_version):
+      logging.warning(
+          'I-Spy expectation \'%s\' already exists, overwriting.',
+          expectation_with_version)
+    screenshots = [self._screenshot_func() for _ in range(8)]
+    logging.info('Generating I-Spy expectation...')
+    self._ispy.GenerateExpectation(expectation_with_version, screenshots)
+
+  def PerformComparison(self, test_run, device_type, expectation,
+                        chrome_version):
+    """Take a screenshot and compare it with the given expectation in I-Spy.
+
+    Args:
+      test_run: name for the test run.
+      device_type: string identifier for the device type.
+      expectation: name for the expectation to compare against.
+      chrome_version: the chrome version as a string of the form "31.0.123.4".
+    """
+    # https://code.google.com/p/chromedriver/issues/detail?id=463
+    time.sleep(1)
+    screenshot = self._screenshot_func()
+    logging.info('Performing I-Spy comparison...')
+    self._ispy.PerformComparison(
+        test_run,
+        self._GetExpectationNameWithVersion(
+            device_type, expectation, chrome_version),
+        screenshot)
+
+  def WaitForUnchangingDOM(self, driver):
+    """Waits for the DOM to stop changing.
+
+    Args:
+      driver: a chromedriver driver instance.
+    """
+    try:
+      driver.execute_async_script(self._wait_for_unchanging_dom_script)
+    except exceptions.TimeoutException:
+      logging.warning('Timed out waiting for DOM to stop changing')
+

chrome/test/functional/ispy/client/chrome_utils_unittest.py

+#!/usr/bin/env python
+#
+# Copyright 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import unittest
+from PIL import Image
+
+import chrome_utils
+from ..common import mock_cloud_bucket
+
+
+class ChromeUtilsTest(unittest.TestCase):
+  """Unittest for ChromeUtils."""
+
+  def setUp(self):
+    self.cloud_bucket = mock_cloud_bucket.MockCloudBucket()
+    self.white_utils = chrome_utils.ChromeUtils(
+        self.cloud_bucket, 'versions.json',
+        lambda: Image.new('RGBA', (10, 10), (255, 255, 255, 255)))
+    self.black_utils = chrome_utils.ChromeUtils(
+        self.cloud_bucket, 'versions.json',
+        lambda: Image.new('RGBA', (10, 10), (0, 0, 0, 255)))
+
+  def testGenerateExpectationsRunComparison(self):
+    self.white_utils.GenerateExpectation('device', 'test', '1.1.1.1')
+    self.white_utils.UpdateExpectationVersion('1.1.1.1')
+    self.white_utils.PerformComparison('test1', 'device', 'test', '1.1.1.1')
+    expect_name = self.white_utils._CreateExpectationName(
+        'device', 'test', '1.1.1.1')
+    self.assertFalse(self.white_utils._ispy.FailureExists('test1', expect_name))
+    self.black_utils.PerformComparison('test2', 'device', 'test', '1.1.1.1')
+    self.assertTrue(self.white_utils._ispy.FailureExists('test2', expect_name))
+
+  def testUpdateExpectationVersion(self):
+    self.white_utils.UpdateExpectationVersion('1.0.0.0')
+    self.white_utils.UpdateExpectationVersion('1.0.4.0')
+    self.white_utils.UpdateExpectationVersion('2.1.5.0')
+    self.white_utils.UpdateExpectationVersion('1.1.5.0')
+    self.white_utils.UpdateExpectationVersion('0.0.0.0')
+    self.white_utils.UpdateExpectationVersion('1.1.5.0')
+    self.white_utils.UpdateExpectationVersion('0.0.0.1')
+    versions = json.loads(self.cloud_bucket.DownloadFile('versions.json'))
+    self.assertEqual(versions,
+        ['2.1.5.0', '1.1.5.0', '1.0.4.0', '1.0.0.0', '0.0.0.1', '0.0.0.0'])
+
+
+if __name__ == '__main__':
+  unittest.main()

chrome/test/functional/ispy/common/ispy_utils.py

@@ -86,7 +86,7 @@ class ISpyUtils(object):
     self.cloud_bucket.UpdateFile(full_path, image_tools.EncodePNG(image))
 
 
-  def UploadExpectation(self, expectation, images):
+  def GenerateExpectation(self, expectation, images):
     """Creates and uploads an expectation to GS from a set of images and name.
 
     This method generates a mask from the uploaded images, then
@@ -102,7 +102,7 @@ class ISpyUtils(object):
         GetExpectationPath(expectation, 'expected.png'), images[0])
     self.UploadImage(GetExpectationPath(expectation, 'mask.png'), mask)
 
-  def RunTest(self, test_run, expectation, actual):
+  def PerformComparison(self, test_run, expectation, actual):
     """Runs an image comparison, and uploads discrepancies to GS.
 
     Args:
@@ -190,7 +190,7 @@ class ISpyUtils(object):
     for path in test_paths:
       self.cloud_bucket.RemoveFile(path)
 
-  def UploadExpectationPinkOut(self, expectation, images, pint_out, rgb):
+  def GenerateExpectationPinkOut(self, expectation, images, pint_out, rgb):
     """Uploads an ispy-test to GS with the pink_out workaround.
 
     Args:

chrome/test/functional/ispy/common/ispy_utils_unittest.py

@@ -74,11 +74,11 @@ class ISpyUtilsUnitTest(unittest.TestCase):
     self.assertEquals(self.bucket.datastore['path/to/image.png'],
                       image_tools.EncodePNG(self.black))
 
-  def testUploadExpectation(self):
+  def testGenerateExpectation(self):
     self.bucket.Reset()
     # Upload some tests to the datastore.
-    self.ispy_utils.UploadExpectation('test', [self.white, self.black])
-    self.ispy_utils.UploadExpectation('test1', [self.black, self.black])
+    self.ispy_utils.GenerateExpectation('test', [self.white, self.black])
+    self.ispy_utils.GenerateExpectation('test1', [self.black, self.black])
     # Confirm that the tests were successfully uploaded.
     self.assertEquals(self.bucket.datastore[
         ispy_utils.GetExpectationPath('test', 'expected.png')],
@@ -93,22 +93,22 @@ class ISpyUtilsUnitTest(unittest.TestCase):
         ispy_utils.GetExpectationPath('test1', 'mask.png')],
         image_tools.EncodePNG(self.black))
 
-  def testRunTest(self):
+  def testPerformComparison(self):
     self.bucket.Reset()
-    self.ispy_utils.UploadExpectation('test1', [self.red, self.red])
-    self.ispy_utils.RunTest('test', 'test1', self.black)
+    self.ispy_utils.GenerateExpectation('test1', [self.red, self.red])
+    self.ispy_utils.PerformComparison('test', 'test1', self.black)
     self.assertEquals(self.bucket.datastore[
         ispy_utils.GetFailurePath('test', 'test1', 'actual.png')],
         image_tools.EncodePNG(self.black))
-    self.ispy_utils.RunTest('test', 'test1', self.red)
+    self.ispy_utils.PerformComparison('test', 'test1', self.red)
     self.assertTrue(self.bucket.datastore.has_key(
             ispy_utils.GetFailurePath('test', 'test1', 'actual.png')))
 
   def testGetExpectation(self):
     self.bucket.Reset()
     # Upload some tests to the datastore
-    self.ispy_utils.UploadExpectation('test1', [self.white, self.black])
-    self.ispy_utils.UploadExpectation('test2', [self.red, self.white])
+    self.ispy_utils.GenerateExpectation('test1', [self.white, self.black])
+    self.ispy_utils.GenerateExpectation('test2', [self.red, self.white])
     test1 = self.ispy_utils.GetExpectation('test1')
     test2 = self.ispy_utils.GetExpectation('test2')
     # Check that GetExpectation gets the appropriate tests.
@@ -126,24 +126,24 @@ class ISpyUtilsUnitTest(unittest.TestCase):
 
   def testExpectationExists(self):
     self.bucket.Reset()
-    self.ispy_utils.UploadExpectation('test1', [self.white, self.black])
-    self.ispy_utils.UploadExpectation('test2', [self.white, self.black])
+    self.ispy_utils.GenerateExpectation('test1', [self.white, self.black])
+    self.ispy_utils.GenerateExpectation('test2', [self.white, self.black])
     self.assertTrue(self.ispy_utils.ExpectationExists('test1'))
     self.assertTrue(self.ispy_utils.ExpectationExists('test2'))
     self.assertFalse(self.ispy_utils.ExpectationExists('test3'))
 
   def testFailureExists(self):
     self.bucket.Reset()
-    self.ispy_utils.UploadExpectation('test1', [self.white, self.white])
-    self.ispy_utils.RunTest('test', 'test1', self.black)
-    self.ispy_utils.RunTest('test', 'test1', self.white)
+    self.ispy_utils.GenerateExpectation('test1', [self.white, self.white])
+    self.ispy_utils.PerformComparison('test', 'test1', self.black)
+    self.ispy_utils.PerformComparison('test', 'test1', self.white)
     self.assertTrue(self.ispy_utils.FailureExists('test', 'test1'))
     self.assertFalse(self.ispy_utils.FailureExists('test', 'test2'))
 
   def testRemoveExpectation(self):
     self.bucket.Reset()
-    self.ispy_utils.UploadExpectation('test1', [self.white, self.white])
-    self.ispy_utils.UploadExpectation('test2', [self.white, self.white])
+    self.ispy_utils.GenerateExpectation('test1', [self.white, self.white])
+    self.ispy_utils.GenerateExpectation('test2', [self.white, self.white])
     self.assertTrue(self.ispy_utils.ExpectationExists('test1'))
     self.assertTrue(self.ispy_utils.ExpectationExists('test2'))
     self.ispy_utils.RemoveExpectation('test1')
@@ -155,9 +155,9 @@ class ISpyUtilsUnitTest(unittest.TestCase):
 
   def testRemoveFailure(self):
     self.bucket.Reset()
-    self.ispy_utils.UploadExpectation('test1', [self.white, self.white])
-    self.ispy_utils.UploadExpectation('test2', [self.white, self.white])
-    self.ispy_utils.RunTest('test', 'test1', self.black)
+    self.ispy_utils.GenerateExpectation('test1', [self.white, self.white])
+    self.ispy_utils.GenerateExpectation('test2', [self.white, self.white])
+    self.ispy_utils.PerformComparison('test', 'test1', self.black)
     self.ispy_utils.RemoveFailure('test', 'test1')
     self.assertFalse(self.ispy_utils.FailureExists('test', 'test1'))
     self.assertTrue(self.ispy_utils.ExpectationExists('test1'))
@@ -167,8 +167,8 @@ class ISpyUtilsUnitTest(unittest.TestCase):
   def testGetFailure(self):
     self.bucket.Reset()
     # Upload a result
-    self.ispy_utils.UploadExpectation('test1', [self.red, self.red])
-    self.ispy_utils.RunTest('test', 'test1', self.black)
+    self.ispy_utils.GenerateExpectation('test1', [self.red, self.red])
+    self.ispy_utils.PerformComparison('test', 'test1', self.black)
     res = self.ispy_utils.GetFailure('test', 'test1')
     # Check that the function correctly got the result.
     self.assertEquals(image_tools.EncodePNG(res.expected),
@@ -184,7 +184,7 @@ class ISpyUtilsUnitTest(unittest.TestCase):
   def testGetAllPaths(self):
     self.bucket.Reset()
     # Upload some tests.
-    self.ispy_utils.UploadExpectation('test1', [self.white, self.black])
+    self.ispy_utils.GenerateExpectation('test1', [self.white, self.black])
     # Check that the function gets all urls matching the prefix.
     self.assertEquals(
         set(self.ispy_utils.GetAllPaths(