Make gpu test expectations conflict checker more strict

Previously a conflict between two expectations for the same test meant
that one expectation's tag list was a subset of the other's tag list.
This CL will change the conflict checker so that it will find conflicts
between expectations when they don't have conflicting tags.

Example of test expectations without conflicts
[ mac ] test1 [ RetryOnFailure ]
[ win ] test1 [ Skip ]

There is no conflict in the example above because they have
different operating systems tags.

Example of test expectations with conflicts
[ android qualcomm-adreno-(tm)-418 ] test2 [ RetryOnFailure ]
[ android opengl ] test2 [ Skip ]

There is a conflict in the example above because they do not have
any conflicting tags. The first test expectatoin can apply to ANGLE
and non-ANGLE configurations on an android device.

Bug: chromium:982363
Change-Id: I90b4df7c9d9ce6fd6347e096eab7d02e9023c6cc
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1699501
Commit-Queue: Rakib Hasan <rmhasan@google.com>
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
Cr-Commit-Position: refs/heads/master@{#677043}

content/test/gpu/gpu_tests/test_expectations/pixel_expectations.txt

@@ -49,11 +49,11 @@ crbug.com/653538 [ win amd-0x6613 ] Pixel_GpuRasterization_BlueBox [ RetryOnFail
 # Skip tests on Android WebView that requires embedding viz clients. This
 # includes offscreen canvas and other features. crbug.com/805739 tracks
 # making this work.
-crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvas2DResizeOnWorker [ Skip ]
+crbug.com/805739 [ android android-webview-instrumentation ] Pixel_OffscreenCanvas2DResizeOnWorker [ Skip ]
 crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasAccelerated2D [ Skip ]
 crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasAccelerated2DWorker [ Skip ]
 crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasTransferAfterStyleResize [ Skip ]
-crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasTransferBeforeStyleResize [ Skip ]
+crbug.com/805739 [ android android-webview-instrumentation ] Pixel_OffscreenCanvasTransferBeforeStyleResize [ Skip ]
 crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasWebGLDefault [ Skip ]
 crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasWebGLDefaultWorker [ Skip ]
 crbug.com/805739 [ android-webview-instrumentation ] Pixel_OffscreenCanvasWebGLPaintAfterResize [ Skip ]
@@ -120,20 +120,20 @@ crbug.com/575305 [ win ] Pixel_WebGLSadCanvas [ Skip ]
 crbug.com/872423 [ mac ] Pixel_WebGLSadCanvas [ Skip ]
 crbug.com/575305 [ android ] Pixel_WebGLSadCanvas [ Skip ]
 
-crbug.com/868596 [ android nvidia ] Pixel_CanvasLowLatencyWebGL [ Failure ]
+crbug.com/868596 [ android android-chromium nvidia ] Pixel_CanvasLowLatencyWebGL [ Failure ]
 
-crbug.com/868596 [ android nvidia ] Pixel_OffscreenCanvasWebGLPaintAfterResize [ Failure ]
+crbug.com/868596 [ android android-chromium nvidia ] Pixel_OffscreenCanvasWebGLPaintAfterResize [ Failure ]
 
 # float16 is not yet supported for accelerated canvases
 crbug.com/948141 Pixel_CanvasDisplaySRGBAccelerated2D [ Skip ]
 
 # Fails on Nexus 5, 6 and 6P
-crbug.com/883500 [ android qualcomm-adreno-(tm)-330 ] Pixel_BackgroundImage [ Skip ]
-crbug.com/883500 [ android qualcomm-adreno-(tm)-420 ] Pixel_BackgroundImage [ Skip ]
-crbug.com/883500 [ android qualcomm-adreno-(tm)-430 ] Pixel_BackgroundImage [ Skip ]
+crbug.com/883500 [ android no-skia-renderer qualcomm-adreno-(tm)-330 ] Pixel_BackgroundImage [ Skip ]
+crbug.com/883500 [ android no-use-vulkan qualcomm-adreno-(tm)-420 ] Pixel_BackgroundImage [ Skip ]
+crbug.com/883500 [ android no-use-vulkan qualcomm-adreno-(tm)-430 ] Pixel_BackgroundImage [ Skip ]
 
 # Flakes on Nexus 5X.
-crbug.com/883500 [ android-chromium qualcomm-adreno-(tm)-418 ] Pixel_BackgroundImage [ RetryOnFailure ]
+crbug.com/883500 [ android-chromium no-use-vulkan qualcomm-adreno-(tm)-418 ] Pixel_BackgroundImage [ RetryOnFailure ]
 
 # We do not have software H.264 decoding on Android, so it can't survive a
 # context loss which results in hardware decoder loss.
@@ -148,7 +148,7 @@ crbug.com/911413 [ mac amd-0x679e ] Pixel_Video_MP4_FourColors_Rot_180 [ Skip ]
 crbug.com/911413 [ mac amd-0x679e ] Pixel_Video_MP4_FourColors_Rot_270 [ Skip ]
 
 # Fails on multiple Android devices.
-crbug.com/927107 [ android ] Pixel_CSS3DBlueBox [ Failure ]
+crbug.com/927107 [ android no-skia-renderer ] Pixel_CSS3DBlueBox [ Failure ]
 
 # Fail on Nexus 5, 5X, 6, 6P, 9 and Shield TV.
 crbug.com/925744 [ android ] Pixel_Video_MP4 [ Skip ]
@@ -176,7 +176,7 @@ crbug.com/929425 [ win nvidia ] Pixel_DirectComposition_ComplexOverlays [ RetryO
 crbug.com/950932 [ android-webview-instrumentation qualcomm-adreno-(tm)-418 ] * [ RetryOnFailure ]
 
 # Failing on Nexus 5
-crbug.com/957714 [ android qualcomm-adreno-(tm)-330 ] Pixel_Canvas2DRedBox [ Failure ]
+crbug.com/957714 [ android no-skia-renderer qualcomm-adreno-(tm)-330 ] Pixel_Canvas2DRedBox [ Failure ]
 crbug.com/900706 [ android qualcomm-adreno-(tm)-330 ] Pixel_CanvasLowLatency2D [ Failure ]
 
 # Failing on Pixel 2 FYI.
@@ -201,16 +201,16 @@ crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_BackgroundImage [ Sk
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_Canvas2DRedBox [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_Canvas2DUntagged [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_CanvasDisplayLinearRGBAccelerated2D [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvas2DResizeOnWorker [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasAccelerated2D [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasAccelerated2DWorker [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasTransferAfterStyleResize [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasTransferBeforeStyleResize [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvas2DResizeOnWorker [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasAccelerated2D [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasAccelerated2DWorker [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasTransferAfterStyleResize [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasTransferBeforeStyleResize [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasTransferToImageBitmap [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasTransferToImageBitmapWorker [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasWebGLDefault [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasWebGLDefaultWorker [ Skip ]
-crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_OffscreenCanvasWebglResizeOnWorker [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasWebGLDefault [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasWebGLDefaultWorker [ Skip ]
+crbug.com/969864 [ android android-chromium skia-renderer use-vulkan ] Pixel_OffscreenCanvasWebglResizeOnWorker [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_WebGLGreenTriangle_AA_Alpha [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_WebGLGreenTriangle_AA_NoAlpha [ Skip ]
 crbug.com/969864 [ android skia-renderer use-vulkan ] Pixel_WebGLGreenTriangle_NoAA_Alpha [ Skip ]

content/test/gpu/gpu_tests/test_expectations/webgl2_conformance_expectations.txt

@@ -556,9 +556,9 @@ crbug.com/950552 [ linux intel ] conformance2/textures/misc/tex-base-level-bug.h
 crbug.com/950552 [ linux intel ] conformance2/textures/misc/tex-3d-size-limit.html [ Failure ]
 
 crbug.com/680675 [ linux intel mesa_lt_17.1.6 ] conformance/extensions/webgl-compressed-texture-astc.html [ Failure ]
-crbug.com/905011 [ linux intel mesa_lt_17.3.9 ] conformance2/textures/misc/tex-subimage3d-pixel-buffer-bug.html [ Failure ]
-crbug.com/666384 [ linux intel mesa_lt_17.3.9 ] deqp/functional/gles3/shadertexturefunction/texturesize.html [ Failure ]
-crbug.com/666384 [ linux intel mesa_lt_17.3.9 ] conformance2/textures/misc/tex-3d-mipmap-levels-intel-bug.html [ Failure ]
+crbug.com/905011 [ linux intel no-angle mesa_lt_17.3.9 ] conformance2/textures/misc/tex-subimage3d-pixel-buffer-bug.html [ Failure ]
+crbug.com/666384 [ linux intel no-angle mesa_lt_17.3.9 ] deqp/functional/gles3/shadertexturefunction/texturesize.html [ Failure ]
+crbug.com/666384 [ linux intel no-angle mesa_lt_17.3.9 ] conformance2/textures/misc/tex-3d-mipmap-levels-intel-bug.html [ Failure ]
 crbug.com/905011 [ linux intel opengl passthrough ] conformance2/textures/misc/tex-subimage3d-pixel-buffer-bug.html [ Failure ]
 
 # Linux AMD only.
@@ -728,7 +728,7 @@ crbug.com/949321 [ android qualcomm ] deqp/functional/gles3/framebufferblit/defa
 # This test is failing on Android Pixel 2 and 3 (Qualcomm)
 # Seems to be an OpenGL ES bug.
 crbug.com/945903 [ android qualcomm ] conformance2/rendering/vertex-id.html [ Failure ]
-crbug.com/695742 [ android qualcomm ] deqp/functional/gles3/multisample.html [ RetryOnFailure ]
+crbug.com/695742 [ android qualcomm no-angle ] deqp/functional/gles3/multisample.html [ RetryOnFailure ]
 
 # This test is flaky but can fail three times in a row so it must be
 # marked as Fail instead of Flaky.

content/test/gpu/gpu_tests/test_expectations/webgl_conformance_expectations.txt

 # tags: [ android chromeos linux mac win win10 win7 ]
-# tags: [ android-chromium android-webview-instrumentation debug release ]
+# tags: [ android-chromium android-webview-instrumentation ]
+# tags: [ debug release ]
 # tags: [ amd amd-0x6613 amd-0x699f intel intel-0xa011 intel-0xa2e nvidia
 #         nvidia-0x1cb3 nvidia-0x2184 nvidia-0xfe9 qualcomm
 #         qualcomm-adreno-(tm)-330 qualcomm-adreno-(tm)-418
@@ -426,7 +427,7 @@ crbug.com/907512 [ android android-chromium ] conformance/textures/video/tex-2d-
 crbug.com/733599 [ android ] conformance/textures/video/tex-2d-alpha-alpha-unsigned_byte.html [ RetryOnFailure ]
 crbug.com/733599 [ android no-angle ] conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html [ RetryOnFailure ]
 crbug.com/733599 [ android no-angle ] conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html [ RetryOnFailure ]
-crbug.com/834933 [ android no-angle ] conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html [ RetryOnFailure ]
+crbug.com/834933 [ android android-chromium no-angle ] conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html [ RetryOnFailure ]
 crbug.com/834933 [ android android-chromium ] conformance/textures/video/tex-2d-rgba-rgba-unsigned_byte.html [ RetryOnFailure ]
 
 # This crashes in Android WebView on the Nexus 6, preventing the
@@ -494,7 +495,7 @@ crbug.com/914631 [ android qualcomm-adreno-(tm)-418 passthrough ] conformance/li
 
 # Nexus 6 (Adreno 420) and 6P (Adreno 430)
 crbug.com/499555 [ android qualcomm-adreno-(tm)-420 ] conformance/context/context-attributes-alpha-depth-stencil-antialias.html [ Failure ]
-crbug.com/611945 [ android qualcomm-adreno-(tm)-420 ] conformance/context/context-size-change.html [ Failure ]
+crbug.com/611945 [ android no-angle qualcomm-adreno-(tm)-420 ] conformance/context/context-size-change.html [ Failure ]
 crbug.com/499555 [ android qualcomm-adreno-(tm)-420 ] conformance/context/premultiplyalpha-test.html [ Failure ]
 crbug.com/611945 [ android qualcomm-adreno-(tm)-420 ] conformance/glsl/bugs/gl-fragcoord-multisampling-bug.html [ Failure ]
 crbug.com/611945 [ android qualcomm-adreno-(tm)-420 ] conformance/glsl/bugs/qualcomm-crash.html [ Failure ]
@@ -545,7 +546,7 @@ crbug.com/845411 [ android ] conformance/glsl/constructors/glsl-construct-mat3.h
 crbug.com/845438 [ android ] conformance/glsl/bugs/sketchfab-lighting-shader-crash.html [ RetryOnFailure ]
 
 # Android ANGLE GLES
-crbug.com/981579 [ android opengles ] conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html [ Failure ]
+crbug.com/981579 [ android android-chromium opengles ] conformance/textures/video/tex-2d-rgb-rgb-unsigned_byte.html [ Failure ]
 crbug.com/906724 [ android opengles ] conformance/textures/video/tex-2d-luminance_alpha-luminance_alpha-unsigned_byte.html [ Failure ]
 crbug.com/906724 [ android opengles ] conformance/textures/video/tex-2d-luminance-luminance-unsigned_byte.html [ Failure ]
 

content/test/gpu/gpu_tests/test_expectations_unittest.py

@@ -60,21 +60,23 @@ def _ConflictLeaksRegression(possible_collision, exp):
   return ''
 
 
-def _IsCollision(s1, s2):
-  # s1 collides with s2 when s1 is a subset of s2
-  # A tag is in both sets if its a generic tag
-  # and its in one set while a specifc tag covered by the generic tag is in
-  # the other set.
-  for tag in s1:
-    if (not tag in s2 and not (
-        tag in GENERIC_CONDITIONS and
-        any(_map_specific_to_generic.get(t, t) == tag for t in s2)) and
-        not _map_specific_to_generic.get(tag, tag) in s2):
+def _IsCollision(s1, s2, tag_sets):
+  # s1 and s2 do not collide when there exists a tag in s1 and tag in s2 that
+  # are from the same tag declaration set and are not equal to each other
+  # and one tag is not a generic tag that covers the other.
+  _generic_and_specific_tags_collide = lambda t1, t2: (
+      t1 in GENERIC_CONDITIONS and t2 in _map_specific_to_generic and
+      _map_specific_to_generic[t2] == t1)
+  for tag_set in tag_sets:
+    for t1, t2 in itertools.product(s1, s2):
+      if (t1 in tag_set and t2 in tag_set and t1 != t2 and
+          not _generic_and_specific_tags_collide(t1, t2) and
+          not _generic_and_specific_tags_collide(t2, t1)):
         return False
   return True
 
 
-def _IsDriverTagDuplicated(s1, s2, test_class):
+def _IsDriverTagDuplicated(s1, s2, test_class, tag_sets):
   if not test_class:
     return False
 
@@ -93,7 +95,7 @@ def _IsDriverTagDuplicated(s1, s2, test_class):
   driver_tag2, other_tags2 = classify_tags(s2)
   if not driver_tag1 and not driver_tag2:
     return False
-  if not _IsCollision(other_tags1, other_tags2):
+  if not _IsCollision(other_tags1, other_tags2, tag_sets):
     return False
   if driver_tag1 == driver_tag2 or (not driver_tag1 or not driver_tag2):
     return True
@@ -173,7 +175,7 @@ def _MapGpuDevicesToVendors(tag_sets):
     if any(gpu in tag_set for gpu in GPU_CONDITIONS):
       _map_specific_to_generic.update(
           {t[0]: t[1] for t in
-           itertools.permutations(tag_set, 2) if (t[0] + '-').startswith(t[1])})
+           itertools.permutations(tag_set, 2) if t[0].startswith(t[1] + '-')})
       break
 
 
@@ -259,9 +261,10 @@ def CheckTestExpectationGlobsForCollision(
       for possible_collision in globs_to_match:
         reason = _ConflictLeaksRegression(possible_collision, match)
         if (reason and
-            (_IsCollision(possible_collision.tags, match.tags)
+            (_IsCollision(possible_collision.tags, match.tags, parser.tag_sets)
                 or _IsDriverTagDuplicated(
-                    possible_collision.tags, match.tags, test_class))):
+                    possible_collision.tags, match.tags, test_class,
+                    parser.tag_sets))):
           if not conflicts_found:
             error_msg += ('\n\nFound conflicts for pattern %s in %s:\n' %
                           (glob, file_name))
@@ -288,14 +291,15 @@ def CheckTestExpectationPatternsForCollision(
   master_conflicts_found = False
   error_msg = ''
   _MapGpuDevicesToVendors(parser.tag_sets)
-
   for exp in parser.expectations:
     tests_to_exps[exp.test].append(exp)
   for pattern, exps in tests_to_exps.items():
     conflicts_found = False
     for e1, e2 in itertools.combinations(exps, 2):
-      if (_IsCollision(e1.tags, e2.tags) or _IsCollision(e2.tags, e1.tags)
-          or _IsDriverTagDuplicated(e1.tags, e2.tags, test_class)):
+      if (_IsCollision(e1.tags, e2.tags, parser.tag_sets) or
+          _IsCollision(e2.tags, e1.tags, parser.tag_sets) or
+          _IsDriverTagDuplicated(
+              e1.tags, e2.tags, test_class, parser.tag_sets)):
         if not conflicts_found:
           error_msg += ('\n\nFound conflicts for test %s in %s:\n' %
                         (pattern, file_name))
@@ -469,6 +473,30 @@ class TestGpuTestExpectationsValidators(unittest.TestCase):
       CheckTestExpectationGlobsForCollision(
           test_expectations, 'test.txt', webgl_conformance_test_class)
 
+  def testCollisionsBetweenAngleAndNonAngleConfiguration(self):
+    test_expectations = '''
+    # tags: [ android ]
+    # tags: [ qualcomm-adreno-(tm)-418 ]
+    # tags: [ opengles ]
+    # results: [ RetryOnFailure Skip ]
+    [ android qualcomm-adreno-(tm)-418 ] a/b/c/d [ RetryOnFailure ]
+    [ android opengles ] a/b/c/d [ Skip ]
+    '''
+    with self.assertRaises(AssertionError):
+      CheckTestExpectationPatternsForCollision(test_expectations, 'test.txt')
+
+  def testCollisionsBetweenAngleAndNonAngleConfigurationUsingGlobs(self):
+    test_expectations = '''
+    # tags: [ android ]
+    # tags: [ qualcomm-adreno-(tm)-418 ]
+    # tags: [ opengles ]
+    # results: [ RetryOnFailure Skip ]
+    [ android qualcomm-adreno-(tm)-418 ] a/b/c* [ Skip ]
+    [ android opengles ] a/b/c/d [ RetryOnFailure ]
+    '''
+    with self.assertRaises(AssertionError):
+      CheckTestExpectationGlobsForCollision(test_expectations, 'test.txt')
+
   def testCollisionInTestExpectationsWithSpecifcAndGenericOsTags(self):
     test_expectations = '''# tags: [ mac win linux xp ]
     # tags: [ intel amd nvidia ]