Add GPU typ tag mismatch script

Adds a script to find cases where the tags known to the tests and the
tags generated when tests are run on the bots are different. This is
a necessary tool for making the generation of unknown tags fatal in the
future, as we need to know what tags we need to ignore or otherwise
address before turning on that functionality.

Also updates a couple of ignored tags that were pointed out by the
script.

Bug: 1011107
Change-Id: I9022fc779de718b53ea3469d6048dd0fcd7b8c73
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2628043
Commit-Queue: Brian Sheedy <bsheedy@chromium.org>
Auto-Submit: Brian Sheedy <bsheedy@chromium.org>
Reviewed-by: Yuly Novikov <ynovikov@chromium.org>
Cr-Commit-Position: refs/heads/master@{#843318}

content/test/gpu/find_unknown_typ_tags.py

+#!/usr/bin/env vpython
+# Copyright 2021 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Script for comparing known typ tags to what's generated on the bots.
+
+This is important for making unknown generated tags fatal, as we need an easy
+way to check if any bots are producing tags we don't know about.
+
+Depends on the `bq` tool, which is available as part of the Google Cloud SDK
+https://cloud.google.com/sdk/docs/quickstarts.
+"""
+
+import argparse
+import json
+import os
+import subprocess
+
+from gpu_tests import path_util
+path_util.SetupTelemetryPaths()
+from gpu_tests import gpu_integration_test
+
+from typ import expectations_parser
+
+BQ_QUERY_TEMPLATE = """\
+WITH
+    tags AS (
+        SELECT (
+            ARRAY(
+                SELECT value
+                FROM tr.tags
+                WHERE key = "typ_tag"
+            )
+        ) AS typ_tags
+        FROM `{table}` tr
+    )
+SELECT DISTINCT typ_tags
+FROM tags
+CROSS JOIN UNNEST(tags.typ_tags) as typ_tags
+"""
+MAX_ROWS = (2**31) - 1
+
+
+def ParseArgs():
+  parser = argparse.ArgumentParser(
+      'Script for finding cases where the typ tags generated on the bots and '
+      'the typ tags we know about are out of sync.')
+  parser.add_argument('--project',
+                      required=True,
+                      help='The billing project to use for BigQuery queries. '
+                      'Must have access to the ResultDB BQ tables, e.g. '
+                      '"luci-resultdb.chromium.gpu_ci_test_results".')
+  return parser.parse_args()
+
+
+def main():
+  args = ParseArgs()
+
+  # Get the list of tags in expectation files. Any expectation file will do
+  # since tags are synced between all of them.
+  expectation_file_path = os.path.join(os.path.dirname(__file__), 'gpu_tests',
+                                       'test_expectations',
+                                       'info_collection_expectations.txt')
+  with open(expectation_file_path) as f:
+    list_parser = expectations_parser.TaggedTestListParser(f.read())
+  used_tags = set()
+  for tag_set in list_parser.tag_sets:
+    used_tags |= set(tag_set)
+
+  # Get the list of ignored tags from the GPU tests.
+  ignored_tags = set(gpu_integration_test.GpuIntegrationTest.IgnoredTags())
+
+  # Get the list of all tags being generated on the bots.
+  generated_tags = set()
+  for table in [
+      'luci-resultdb.chromium.gpu_ci_test_results',
+      'luci-resultdb.chromium.gpu_try_test_results'
+  ]:
+    query = BQ_QUERY_TEMPLATE.format(table=table)
+    cmd = [
+        'bq',
+        'query',
+        '--max_rows=%d' % MAX_ROWS,
+        '--format=json',
+        '--project_id=%s' % args.project,
+        '--use_legacy_sql=false',
+        query,
+    ]
+    with open(os.devnull, 'w') as devnull:
+      try:
+        stdout = subprocess.check_output(cmd, stderr=devnull)
+      except subprocess.CalledProcessError as e:
+        print e.output
+        raise
+    results = json.loads(stdout)
+    for pair in results:
+      generated_tags |= set(pair.values())
+
+  known_tags = used_tags | ignored_tags
+
+  unused_tags = generated_tags - known_tags
+  if unused_tags:
+    print 'Tags that were generated but unused:'
+    for t in unused_tags:
+      print t
+    print ''
+
+  stale_tags = known_tags - generated_tags
+  if stale_tags:
+    print 'Tags that are known but not generated:'
+    for t in stale_tags:
+      print t
+
+  if not (unused_tags or stale_tags):
+    print 'Known and generated tags are in sync.'
+
+
+if __name__ == '__main__':
+  main()

content/test/gpu/gpu_tests/gpu_integration_test.py

@@ -602,11 +602,13 @@ class GpuIntegrationTest(
         'qualcomm-adreno-(tm)-420',  # android-nexus-6
         'qualcomm-adreno-(tm)-540',  # android-pixel-2
         'nvidia-nvidia-tegra',  # android-nexus-9 and android-shield-android-tv
-        'vmware',  # VMs
+        'vmware,',  # VMs
         'vmware,-0x1050',  # ChromeOS VMs
         # Fuchsia VMs
         ('google-angle-(vulkan-1.1.0(swiftshader-device-('
          'llvm-7.0.1)-(0x0000c0de)))'),
+        ('google-angle-(vulkan-1.1.0(swiftshader-device-('
+         'llvm-10.0.0)-(0x0000c0de)))'),
         # These browsers are analogous to a particular OS, and specifying the
         # OS name is clearer.
         'cros-chrome',  # ChromeOS