Allow autotest to collect tests that don't include gtest files

Autotest checks for gtest files to better determine if a give file is
actually a test file or not. However, because many files don't actually
include what they use, many test files don't directly include gtest.h,
gmock.h, or a test_utils.h file. Instead, they may include those via
something like sim_test.h, resulting in the script ignoring those files
right now.

This change improves that by still looking for files that include a
gtest file, but until it has found at least one file that does, it
will also collect files that match the test file name regex and do not
match the gtest include regex. Then, if no files are found with the
gtest includes, it will move forward under the assumption that the
file it found without the gtest include was the correct test file.

It will still prefer files with gtest includes, but has a fallback in
case none exist, providing a best effort chance at building and
running the tests.

Also includes a small improvement to outputting the file names that
it finds after a recursive file search.

Change-Id: Ia56792426b55cb17a43ffe2545b4deefb5dba0dd
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2568710
Commit-Queue: Mario Bianucci <mabian@microsoft.com>
Reviewed-by: Michael Thiessen <mthiesse@chromium.org>
Cr-Commit-Position: refs/heads/master@{#832607}

tools/autotest.py

@@ -33,6 +33,7 @@ import re
 import subprocess
 import sys
 
+from enum import Enum
 from pathlib import Path
 
 USE_PYTHON_3 = f'This script will only run under python3.'
@@ -68,20 +69,28 @@ def ExitWithMessage(*args):
   print(*args, file=sys.stderr)
   sys.exit(1)
 
+
+class TestValidity(Enum):
+  NOT_A_TEST = 0  # Does not match test file regex.
+  MAYBE_A_TEST = 1  # Matches test file regex, but doesn't include gtest files.
+  VALID_TEST = 2  # Matches test file regex and includes gtest files.
+
+
 def IsTestFile(file_path):
   if not TEST_FILE_NAME_REGEX.match(file_path):
-    return False
+    return TestValidity.NOT_A_TEST
   if file_path.endswith('.cc'):
     # Try a bit harder to remove non-test files for c++. Without this,
     # 'autotest.py base/' finds non-test files.
     try:
       with open(file_path, 'r', encoding='utf-8') as f:
         if GTEST_INCLUDE_REGEX.search(f.read()) is not None:
-          return True
+          return TestValidity.VALID_TEST
     except IOError:
       pass
-    return False
-  return True
+    # It may still be a test file, even if it doesn't include a gtest file.
+    return TestValidity.MAYBE_A_TEST
+  return TestValidity.VALID_TEST
 
 
 class CommandError(Exception):
@@ -138,27 +147,33 @@ def BuildTestTargetsWithNinja(out_dir, targets, dry_run):
 def RecursiveMatchFilename(folder, filename):
   current_dir = os.path.split(folder)[-1]
   if current_dir.startswith('out') or current_dir.startswith('.'):
-    return []
-  matches = []
+    return [[], []]
+  exact = []
+  close = []
   with os.scandir(folder) as it:
     for entry in it:
       if (entry.is_symlink()):
         continue
       if (entry.is_file() and filename in entry.path and
           not os.path.basename(entry.path).startswith('.')):
-        if IsTestFile(entry.path):
-          matches.append(entry.path)
+        file_validity = IsTestFile(entry.path)
+        if file_validity is TestValidity.VALID_TEST:
+          exact.append(entry.path)
+        elif file_validity is TestValidity.MAYBE_A_TEST:
+          close.append(entry.path)
       if entry.is_dir():
         # On Windows, junctions are like a symlink that python interprets as a
         # directory, leading to exceptions being thrown. We can just catch and
         # ignore these exceptions like we would ignore symlinks.
         try:
-          matches += RecursiveMatchFilename(entry.path, filename)
+          matches = RecursiveMatchFilename(entry.path, filename)
+          exact += matches[0]
+          close += matches[1]
         except FileNotFoundError as e:
           if DEBUG:
             print(f'Failed to scan directory "{entry}" - junction?')
           pass
-  return matches
+  return [exact, close]
 
 
 def FindTestFilesInDirectory(directory):
@@ -168,10 +183,13 @@ def FindTestFilesInDirectory(directory):
   for root, dirs, files in os.walk(directory):
     for f in files:
       path = os.path.join(root, f)
-      if IsTestFile(path):
+      file_validity = IsTestFile(path)
+      if file_validity is TestValidity.VALID_TEST:
         if DEBUG:
           print(path)
         test_files.append(path)
+      elif DEBUG and file_validity is TestValidity.MAYBE_A_TEST:
+        print(path + ' matched but doesn\'t include gtest files, skipping.')
   return test_files
 
 
@@ -180,10 +198,20 @@ def FindMatchingTestFiles(target):
   if os.path.isfile(target):
     # If the target is a C++ implementation file, try to guess the test file.
     if target.endswith('.cc') or target.endswith('.h'):
-      if IsTestFile(target):
+      target_validity = IsTestFile(target)
+      if target_validity is TestValidity.VALID_TEST:
         return [target]
       alternate = f"{target.rsplit('.', 1)[0]}_unittest.cc"
-      if os.path.isfile(alternate) and IsTestFile(alternate):
+      alt_validity = TestValidity.NOT_A_TEST if not os.path.isfile(
+          alternate) else IsTestFile(alternate)
+      if alt_validity is TestValidity.VALID_TEST:
+        return [alternate]
+
+      # If neither the target nor its alternative were valid, check if they just
+      # didn't include the gtest files before deciding to exit.
+      if target_validity is TestValidity.MAYBE_A_TEST:
+        return [target]
+      if alt_validity is TestValidity.MAYBE_A_TEST:
         return [alternate]
       ExitWithMessage(f"{target} doesn't look like a test file")
     return [target]
@@ -202,18 +230,26 @@ def FindMatchingTestFiles(target):
     target = target.replace(os.path.altsep, os.path.sep)
   if DEBUG:
     print('Finding files with full path containing: ' + target)
-  results = RecursiveMatchFilename(SRC_DIR, target)
+
+  [exact, close] = RecursiveMatchFilename(SRC_DIR, target)
   if DEBUG:
-    print('Found matching file(s): ' + ' '.join(results))
-  if len(results) > 1:
+    if exact:
+      print('Found exact matching file(s):')
+      print('\n'.join(exact))
+    if close:
+      print('Found possible matching file(s):')
+      print('\n'.join(close))
+
+  test_files = exact if len(exact) > 0 else close
+  if len(test_files) > 1:
     # Arbitrarily capping at 10 results so we don't print the name of every file
     # in the repo if the target is poorly specified.
-    results = results[:10]
+    test_files = test_files[:10]
     ExitWithMessage(f'Target "{target}" is ambiguous. Matching files: '
-                    f'{results}')
-  if not results:
+                    f'{test_files}')
+  if not test_files:
     ExitWithMessage(f'Target "{target}" did not match any files.')
-  return results
+  return test_files
 
 
 def IsTestTarget(target):