Added WebGL conformance test expectation script.

Using the WebKit LayoutTests as a template, you can now specify slow or failing
tests to the header generating script.


Review URL: http://codereview.chromium.org/7563021

git-svn-id: svn://svn.chromium.org/chrome/trunk/src@95510 0039d316-1c4b-4281-b951-d872f2087c98

chrome/chrome_tests.gypi

@@ -3548,6 +3548,7 @@
         {
           'action_name': 'generate_webgl_conformance_test_list',
           'inputs': [
+            'test/gpu/webgl_conformance_test_expectations.txt',
             'test/gpu/generate_webgl_conformance_test_list.py',
             '<!@(python test/gpu/generate_webgl_conformance_test_list.py --input)',
           ],

chrome/test/gpu/generate_webgl_conformance_test_list.py

@@ -11,8 +11,6 @@ for individual conformance tests (each on a new line). It recursively parses
 sent to the C++ header file.
 """
 
-__author__ = 'jrt@chromium.org (Joe Tessler)'
-
 import getopt
 import os
 import re
@@ -41,6 +39,27 @@ HEADER_GUARD_END = """
 # Assume this script is run from the src/chrome/ directory.
 INPUT_DIR = "../third_party/webgl_conformance"
 INPUT_FILE = "00_test_list.txt"
+EXPECTATION_FILE = "test/gpu/webgl_conformance_test_expectations.txt"
+EXPECTATION_REGEXP = re.compile(
+    r'^(?P<BUG>\S+)\s+'
+     '(?P<OS>(\s*(WIN|MAC|LINUX)\s*)+):'
+     '(?P<TEST>[^=]+)='
+     '(?P<OUTCOME>(\s*(PASS|FAIL|TIMEOUT)\s*)+)')
+
+def is_matching_os(expected_os_list):
+  """Returns true if the current OS is in the given list.
+
+  Given a list containing 'WIN', 'MAC' or 'LINUX', return true if the current
+  OS, represented as 'win32', 'darwin' or 'linux*', respectively, exists in the
+  list.
+  """
+  if sys.platform.startswith('linux') and 'LINUX' in expected_os_list:
+    return True;
+  if sys.platform == 'darwin' and 'MAC' in expected_os_list:
+    return True;
+  if sys.platform == 'win32' and 'WIN' in expected_os_list:
+    return True;
+  return False;
 
 def main(argv):
   """Main function for the WebGL conformance test list generator.
@@ -72,6 +91,27 @@ def main(argv):
     print >> sys.stderr, "ERROR: WebGL conformance tests do not exist."
     return 1
 
+  test_prefix = {}
+  if os.path.exists(EXPECTATION_FILE):
+    test_expectations = open(EXPECTATION_FILE)
+    for line in test_expectations:
+      line_match = EXPECTATION_REGEXP.match(line)
+      if line_match:
+        match_dict = line_match.groupdict()
+        os_list = match_dict['OS'].strip().split()
+        if not is_matching_os(os_list):
+          continue
+        test = match_dict['TEST'].strip()
+        outcome_list = match_dict['OUTCOME'].strip().split()
+        if 'TIMEOUT' in outcome_list:
+          test_prefix[test] = "DISABLED_"
+        elif 'FAIL' in outcome_list:
+          if 'PASS' in outcome_list:
+            test_prefix[test] = "FLAKY_"
+          else:
+            test_prefix[test] = "FAILS_"
+    test_expectations.close()
+
   output = open(output_file, "w")
   output.write(COPYRIGHT)
   output.write(WARNING)
@@ -106,6 +146,9 @@ def main(argv):
         # is sent through javascript.
         url = "%s/%s" % (os.path.dirname(filename), url)
         if os.path.exists(os.path.join(INPUT_DIR, url)):
+          # Append "DISABLED_" or "FAILS_" if needed.
+          if name in test_prefix:
+            name = test_prefix[name] + name
           output.write('CONFORMANCE_TEST(%s,\n  "%s");\n' % (name, url))
         else:
           print >> sys.stderr, "WARNING: %s does not exist (skipped)." % url

chrome/test/gpu/webgl_conformance_test_expectations.txt

+// Copyright (c) 2011 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This file contains a list of defective WebGL conformance tests. The expected
+// format is:
+//  {BUG#} {MAC,WIN,LINUX} : {TEST_NAME} = {FAIL,TIMEOUT}
+//
+// Examples:
+//  91530 MAC WIN LINUX : context_lost_restored = TIMEOUT
+//    This will mark the test as DISABLED_*.
+//
+//  91533 WIN : gl_min_uniforms = FAIL
+//    This will mark the test as FAILS_*.
+
+// -----------------------------------------------------------------------------
+// SLOW TESTS
+// -----------------------------------------------------------------------------
+
+91530 MAC WIN LINUX : context_lost_restored = TIMEOUT
+91530 LINUX MAC : conformance_quickCheckAPI = TIMEOUT
+91530 WIN : drawingbuffer_static_canvas_test = TIMEOUT
+
+// -----------------------------------------------------------------------------
+// FAILING TESTS
+// -----------------------------------------------------------------------------
+
+91532 LINUX : uniform_location = FAIL  // Probably a driver error.
+91533 WIN : gl_min_uniforms = FAIL
+91533 WIN : oes_standard_derivatives = FAIL
+91533 WIN : texture_npot = FAIL