Add unit tests for BaseWptScriptAdapter in /testing/script/wpt_common.py

This requires adding a PRESUBMIT.py to /testing/scripts to run the unit tests.

Includes some small updates to wpt_common, mainly:
- ensuring crashlot artifact path is added to output json
- replacing os.path.exists with FileSystem.exists

Bug: 1131478

Change-Id: Ia470f57b43709d20825d7f91017647b85d71ad48
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2422346
Commit-Queue: Luke Z <lpz@chromium.org>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Reviewed-by: Stephen McGruer <smcgruer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#809757}

testing/scripts/OWNERS

@@ -10,9 +10,9 @@ per-file check_static_initializers.py=thomasanderson@chromium.org
 
 per-file run_wpt_tests.py=lpz@chromium.org
 per-file run_wpt_tests.py=robertma@chromium.org
-per-file wpt_common.py=lpz@chromium.org
-per-file wpt_common.py=robertma@chromium.org
-per-file wpt_common.py=rmhasan@chromium.org
+per-file wpt_common*.py=lpz@chromium.org
+per-file wpt_common*.py=robertma@chromium.org
+per-file wpt_common*.py=rmhasan@chromium.org
 
 per-file run_performance_tests.py=johnchen@chromium.org
 per-file run_performance_tests.py=wenbinzhang@google.com

testing/scripts/PRESUBMIT.py

+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Top-level presubmit script for testing/trigger_scripts.
+
+See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
+for more details about the presubmit API built into depot_tools.
+"""
+
+def CommonChecks(input_api, output_api):
+  return input_api.canned_checks.RunUnitTestsInDirectory(
+      input_api, output_api, '.', whitelist=[r'^.+_unittest\.py$'])
+
+def CheckChangeOnUpload(input_api, output_api):
+  return CommonChecks(input_api, output_api)
+
+def CheckChangeOnCommit(input_api, output_api):
+  return CommonChecks(input_api, output_api)

testing/scripts/get_compile_targets.py

@@ -30,8 +30,10 @@ def main(argv):
     if filename in ('common.py',
                     'get_compile_targets.py',
                     'gpu_integration_test_adapter.py',
+                    'PRESUBMIT.py',
                     'sizes_common.py',
-                    'wpt_common.py'):
+                    'wpt_common.py',
+                    'wpt_common_unittest.py'):
       continue
 
     with common.temporary_file() as tempfile_path:

testing/scripts/wpt_common.py

@@ -26,12 +26,16 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
     as integrating output with the results viewer. Subclasses contain other
     (usually platform-specific) logic."""
 
-    def __init__(self):
+    def __init__(self, host=None):
         super(BaseWptScriptAdapter, self).__init__()
-        self.fs = FileSystem()
+        if not host:
             host = Host()
+        self.fs = host.filesystem
         self.port = host.port_factory.get()
         self.wpt_manifest = self.port.wpt_manifest("external/wpt")
+        # Path to the output of the test run. Comes from the args passed to the
+        # run, parsed after this constructor. Can be overwritten by tests.
+        self.wpt_output = None
 
     def generate_test_output_args(self, output):
         return ['--log-chromium', output]
@@ -47,24 +51,27 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
                 '--chunk-type=hash']
 
     def do_post_test_run_tasks(self):
+        if not self.wpt_output and self.options:
+            self.wpt_output = self.options.isolated_script_test_output
+
         # Move json results into layout-test-results directory
-        results_dir = os.path.dirname(self.options.isolated_script_test_output)
+        results_dir = os.path.dirname(self.wpt_output)
         layout_test_results = os.path.join(results_dir, 'layout-test-results')
-        if os.path.exists(layout_test_results):
+        if self.fs.exists(layout_test_results):
             self.fs.rmtree(layout_test_results)
         self.fs.maybe_make_directory(layout_test_results)
 
         # Perform post-processing of wptrunner output
         self.process_wptrunner_output()
 
-        self.fs.copyfile(self.options.isolated_script_test_output,
+        self.fs.copyfile(self.wpt_output,
                         os.path.join(layout_test_results, 'full_results.json'))
         # create full_results_jsonp.js file which is used to
         # load results into the results viewer
         self.fs.write_text_file(
             os.path.join(layout_test_results, 'full_results_jsonp.js'),
             'ADD_FULL_RESULTS(%s);' % self.fs.read_text_file(
-                self.options.isolated_script_test_output))
+                self.wpt_output))
 
         # copy layout test results viewer to layout-test-results directory
         self.fs.copyfile(
@@ -78,13 +85,13 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
         or artifacts which need to be extracted into their own files and removed
         from the json file (to avoid duplication)."""
         output_json = json.loads(
-            self.fs.read_text_file(self.options.isolated_script_test_output))
+            self.fs.read_text_file(self.wpt_output))
         test_json = output_json["tests"]
-        results_dir = os.path.dirname(self.options.isolated_script_test_output)
+        results_dir = os.path.dirname(self.wpt_output)
         self._process_test_leaves(results_dir, output_json["path_delimiter"],
                                   test_json, "")
         # Write output_json back to the same file after modifying it in memory
-        self.fs.write_text_file(self.options.isolated_script_test_output,
+        self.fs.write_text_file(self.wpt_output,
                                 json.dumps(output_json))
 
     def _process_test_leaves(self, results_dir, delim, root_node, path_so_far):
@@ -131,6 +138,8 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
                 artifact_subpath = self._write_log_artifact(
                     test_failures.FILENAME_SUFFIX_CRASH_LOG,
                     results_dir, path_so_far, crashlog_artifact)
+                if artifact_subpath:
+                    root_node["artifacts"]["crash_log"] = [artifact_subpath]
 
             return
 
@@ -169,7 +178,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
 
         test_file_path = os.path.join(EXTERNAL_WPT_TESTS_DIR, test_file_subpath)
         expected_ini_path = test_file_path + ".ini"
-        if not os.path.exists(expected_ini_path):
+        if not self.fs.exists(expected_ini_path):
             return None
 
         # This test has checked-in expected output. It needs to be copied to the
@@ -206,7 +215,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
         )
         log_artifact_full_path = os.path.join(results_dir,
                                               log_artifact_sub_path)
-        if not os.path.exists(os.path.dirname(log_artifact_full_path)):
+        if not self.fs.exists(os.path.dirname(log_artifact_full_path)):
             self.fs.maybe_make_directory(
                 os.path.dirname(log_artifact_full_path))
         self.fs.write_text_file(log_artifact_full_path,
@@ -257,7 +266,7 @@ class BaseWptScriptAdapter(common.BaseIsolatedScriptArgsAdapter):
             result[screenshot_key] = screenshot_sub_path
 
             screenshot_full_path = os.path.join(results_dir,screenshot_sub_path)
-            if not os.path.exists(os.path.dirname(screenshot_full_path)):
+            if not self.fs.exists(os.path.dirname(screenshot_full_path)):
                 self.fs.maybe_make_directory(
                     os.path.dirname(screenshot_full_path))
             # Note: we are writing raw bytes to this file

testing/scripts/wpt_common_unittest.py

+#!/usr/bin/env vpython
+# Copyright (c) 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Unit tests for common functionality of wpt testing scripts."""
+
+import base64
+import json
+import os
+import unittest
+
+from wpt_common import (
+    BaseWptScriptAdapter, EXTERNAL_WPT_TESTS_DIR, WEB_TESTS_DIR
+)
+
+from blinkpy.common.host_mock import MockHost
+from blinkpy.web_tests.port.factory_mock import MockPortFactory
+from blinkpy.w3c.wpt_manifest import BASE_MANIFEST_NAME
+
+# The path where the output of a wpt run was written. This is the file that
+# gets processed by BaseWptScriptAdapter.
+OUTPUT_JSON_FILENAME = "/out.json"
+
+
+class BaseWptScriptAdapterTest(unittest.TestCase):
+    def setUp(self):
+        self.host = MockHost()
+        self.host.port_factory = MockPortFactory(self.host)
+        self.port = self.host.port_factory.get()
+
+        # Create a testing manifest containing any test files that we
+        # might interact with.
+        self.host.filesystem.write_text_file(
+            self.port.web_tests_dir() + '/external/' + BASE_MANIFEST_NAME,
+            json.dumps({
+                'items': {
+                    'reftest': {
+                        'reftest.html': [
+                            'c3f2fb6f436da59d43aeda0a7e8a018084557033',
+                            [None, [['reftest-ref.html', '==']], {}],
+                        ]
+                    },
+                    'testharness': {
+                        'test.html': [
+                            'd933fd981d4a33ba82fb2b000234859bdda1494e',
+                            [None, {}]
+                        ],
+                        'crash.html': [
+                            'd933fd981d4a33ba82fb2b000234859bdda1494e',
+                            [None, {}]
+                        ],
+                        'variant.html': [
+                            'b8db5972284d1ac6bbda0da81621d9bca5d04ee7',
+                            ['variant.html?foo=bar/abc', {}],
+                            ['variant.html?foo=baz', {}],
+                        ],
+                        'dir': {
+                            'multiglob.https.any.js': [
+                                'd6498c3e388e0c637830fa080cca78b0ab0e5305',
+                                ['dir/multiglob.https.any.window.html', {}],
+                                ['dir/multiglob.https.any.worker.html', {}],
+                            ],
+                        },
+                    },
+                },
+            }))
+        self.host.filesystem.write_text_file(
+            os.path.join(WEB_TESTS_DIR, "fast", "harness", "results.html"),
+            "results-viewer-body")
+        self.wpt_adapter = BaseWptScriptAdapter(self.host)
+        self.wpt_adapter.wpt_output = OUTPUT_JSON_FILENAME
+
+    def _create_json_output(self, json_dict):
+        """Writing some json output for processing."""
+        self.host.filesystem.write_text_file(OUTPUT_JSON_FILENAME,
+                                             json.dumps(json_dict))
+
+    def _load_json_output(self):
+        """Loads the json output after post-processing."""
+        return json.loads(self.host.filesystem.read_text_file(
+            OUTPUT_JSON_FILENAME))
+
+    def test_write_log_artifact(self):
+        # Ensure that log artifacts are written to the correct location.
+        json_dict = {
+            'tests': {
+                'test.html': {
+                    'expected': 'PASS',
+                    'actual': 'PASS',
+                    'artifacts': {
+                        'wpt_actual_status': ['OK'],
+                        'log': ['test.html actual text'],
+                    },
+                },
+            },
+            'path_delimiter': '/',
+        }
+        self._create_json_output(json_dict)
+        self.wpt_adapter.do_post_test_run_tasks()
+        written_files = self.wpt_adapter.fs.written_files
+        self.assertEqual(
+            "test.html actual text",
+            written_files["/layout-test-results/test-actual.txt"])
+        # Ensure the artifact in the json was replaced with the location of
+        # the newly-created file.
+        updated_json = self._load_json_output()
+        self.assertFalse(
+            "log" in updated_json["tests"]["test.html"]["artifacts"])
+        self.assertEqual(
+            ["layout-test-results/test-actual.txt"],
+            updated_json["tests"]["test.html"]["artifacts"]["actual_text"])
+
+    def test_write_crashlog_artifact(self):
+        # Ensure that crash log artifacts are written to the correct location.
+        json_dict = {
+            'tests': {
+                'test.html': {
+                    'expected': 'PASS',
+                    'actual': 'CRASH',
+                    'artifacts': {
+                        'wpt_actual_status': ['CRASH'],
+                        'wpt_crash_log': ['test.html crashed!'],
+                    },
+                },
+            },
+            'path_delimiter': '/',
+        }
+        self._create_json_output(json_dict)
+        self.wpt_adapter.do_post_test_run_tasks()
+        written_files = self.wpt_adapter.fs.written_files
+        self.assertEqual(
+            "test.html crashed!",
+            written_files["/layout-test-results/test-crash-log.txt"])
+        # Ensure the artifact in the json was replaced with the location of
+        # the newly-created file.
+        updated_json = self._load_json_output()
+        self.assertFalse(
+            "wpt_crash_log" in updated_json["tests"]["test.html"]["artifacts"])
+        self.assertEqual(
+            ["layout-test-results/test-crash-log.txt"],
+            updated_json["tests"]["test.html"]["artifacts"]["crash_log"])
+
+    def test_write_screenshot_artifacts(self):
+        # Ensure that screenshots are written to the correct filenames and
+        # their bytes are base64 decoded.
+        json_dict = {
+            'tests': {
+                'reftest.html': {
+                    'expected': 'PASS',
+                    'actual': 'PASS',
+                    'artifacts': {
+                        'wpt_actual_status': ['PASS'],
+                        'screenshots': [
+                            'reftest.html:abcd',
+                            'reftest-ref.html:bcde',
+                        ],
+                    },
+                },
+            },
+            'path_delimiter': '/',
+        }
+        self._create_json_output(json_dict)
+        self.wpt_adapter.do_post_test_run_tasks()
+        written_files = self.wpt_adapter.fs.written_files
+        self.assertEqual(
+            base64.b64decode('abcd'),
+            written_files["/layout-test-results/reftest-actual.png"])
+        self.assertEqual(
+            base64.b64decode('bcde'),
+            written_files["/layout-test-results/reftest-expected.png"])
+        # Ensure the artifacts in the json were replaced with the location of
+        # the newly-created files.
+        updated_json = self._load_json_output()
+        self.assertFalse(
+            "screenshots" in updated_json["tests"]["reftest.html"]["artifacts"])
+        self.assertEqual(
+            ["layout-test-results/reftest-actual.png"],
+            updated_json["tests"]["reftest.html"]["artifacts"]["actual_image"])
+        self.assertEqual(
+            ["layout-test-results/reftest-expected.png"],
+            updated_json["tests"]["reftest.html"]["artifacts"]
+                ["expected_image"])
+
+    def test_copy_expected_output(self):
+        # Check that an -expected.txt file is created from a checked-in metadata
+        # ini file if it exists for a test
+        json_dict = {
+            'tests': {
+                'test.html': {
+                    'expected': 'PASS',
+                    'actual': 'PASS',
+                    'artifacts': {
+                        'wpt_actual_status': ['OK'],
+                        'log': ['test.html actual text'],
+                    },
+                },
+            },
+            'path_delimiter': '/',
+        }
+        self._create_json_output(json_dict)
+        # Also create a checked-in metadata file for this test
+        self.host.filesystem.write_text_file(
+            os.path.join(EXTERNAL_WPT_TESTS_DIR, "test.html.ini"),
+            "test.html checked-in metadata")
+        self.wpt_adapter.do_post_test_run_tasks()
+        written_files = self.wpt_adapter.fs.written_files
+        self.assertEqual(
+            "test.html actual text",
+            written_files["/layout-test-results/test-actual.txt"])
+        # The checked-in metadata file gets renamed from .ini to -expected.txt
+        self.assertEqual(
+            "test.html checked-in metadata",
+            written_files["/layout-test-results/test-expected.txt"])
+        # Ensure the artifacts in the json were replaced with the locations of
+        # the newly-created files.
+        updated_json = self._load_json_output()
+        self.assertFalse(
+            "log" in updated_json["tests"]["test.html"]["artifacts"])
+        self.assertEqual(
+            ["layout-test-results/test-actual.txt"],
+            updated_json["tests"]["test.html"]["artifacts"]["actual_text"])
+        self.assertEqual(
+            ["layout-test-results/test-expected.txt"],
+            updated_json["tests"]["test.html"]["artifacts"]["expected_text"])
+
+    def test_expected_output_for_variant(self):
+        # Check that an -expected.txt file is created from a checked-in metadata
+        # ini file for a variant test. Variants are a little different because
+        # we have to use the manifest to map a test name to the test file, and
+        # determine the associated metadata from the test file.
+        # Check that an -expected.txt file is created from a checked-in metadata
+        # ini file if it exists for a test
+        json_dict = {
+            'tests': {
+                'variant.html?foo=bar/abc': {
+                    'expected': 'PASS',
+                    'actual': 'PASS',
+                    'artifacts': {
+                        'wpt_actual_status': ['OK'],
+                        'log': ['variant bar/abc actual text'],
+                    },
+                },
+            },
+            'path_delimiter': '/',
+        }
+        self._create_json_output(json_dict)
+        # Also create a checked-in metadata file for this test. This filename
+        # matches the test *file* name, not the test name (which includes the
+        # variant).
+        self.host.filesystem.write_text_file(
+            os.path.join(EXTERNAL_WPT_TESTS_DIR, "variant.html.ini"),
+            "variant.html checked-in metadata")
+        self.wpt_adapter.do_post_test_run_tasks()
+        written_files = self.wpt_adapter.fs.written_files
+        self.assertEqual(
+            "variant bar/abc actual text",
+            written_files[
+                "/layout-test-results/variant_foo=bar_abc-actual.txt"])
+        # The checked-in metadata file gets renamed from .ini to -expected.txt
+        self.assertEqual(
+            "variant.html checked-in metadata",
+            written_files[
+                "/layout-test-results/variant_foo=bar_abc-expected.txt"])
+        # Ensure the artifacts in the json were replaced with the locations of
+        # the newly-created files.
+        updated_json = self._load_json_output()
+        self.assertFalse("log" in updated_json["tests"]
+                         ["variant.html?foo=bar/abc"]["artifacts"])
+        self.assertEqual(
+            ["layout-test-results/variant_foo=bar_abc-actual.txt"],
+            updated_json["tests"]["variant.html?foo=bar/abc"]["artifacts"]
+                ["actual_text"])
+        self.assertEqual(
+            ["layout-test-results/variant_foo=bar_abc-expected.txt"],
+            updated_json["tests"]["variant.html?foo=bar/abc"]["artifacts"]
+                ["expected_text"])
+
+    def test_expected_output_for_multiglob(self):
+        # Check that an -expected.txt file is created from a checked-in metadata
+        # ini file for a multiglobal test. Multi-globals are a little different
+        # because we have to use the manifest to map a test name to the test
+        # file, and determine the associated metadata from the test file.
+        json_dict = {
+            'tests': {
+                'dir/multiglob.https.any.worker.html': {
+                    'expected': 'PASS',
+                    'actual': 'PASS',
+                    'artifacts': {
+                        'wpt_actual_status': ['OK'],
+                        'log': ['dir/multiglob worker actual text'],
+                    },
+                },
+            },
+            'path_delimiter': '/',
+        }
+        self._create_json_output(json_dict)
+        # Also create a checked-in metadata file for this test. This filename
+        # matches the test *file* name, not the test name (which includes test
+        # scope).
+        self.host.filesystem.write_text_file(
+            os.path.join(EXTERNAL_WPT_TESTS_DIR,
+                         "dir/multiglob.https.any.js.ini"),
+            "dir/multiglob checked-in metadata")
+        self.wpt_adapter.do_post_test_run_tasks()
+        written_files = self.wpt_adapter.fs.written_files
+        self.assertEqual(
+            "dir/multiglob worker actual text",
+            written_files[
+                "/layout-test-results/dir/"
+                "multiglob.https.any.worker-actual.txt"])
+        # The checked-in metadata file gets renamed from .ini to -expected.txt
+        self.assertEqual(
+            "dir/multiglob checked-in metadata",
+            written_files[
+                "/layout-test-results/dir/"
+                "multiglob.https.any.worker-expected.txt"])
+        # Ensure the artifacts in the json were replaced with the locations of
+        # the newly-created files.
+        updated_json = self._load_json_output()
+        self.assertFalse("log" in updated_json["tests"]
+        ["dir/multiglob.https.any.worker.html"]["artifacts"])
+        self.assertEqual(
+            ["layout-test-results/dir/multiglob.https.any.worker-actual.txt"],
+            updated_json["tests"]["dir/multiglob.https.any.worker.html"]
+            ["artifacts"]["actual_text"])
+        self.assertEqual(
+            ["layout-test-results/dir/multiglob.https.any.worker-expected.txt"],
+            updated_json["tests"]["dir/multiglob.https.any.worker.html"]
+            ["artifacts"]["expected_text"])
+
+if __name__ == '__main__':
+    unittest.main()