[blinkpy] Add tests to prevent configuration mismatch

base.Port has some constants that need to be updated when e.g. concrete
ports get new versions, which is easy to forget. Add unit tests to catch
this and fix two existing issues (a typo and a missing version).

Change-Id: I17dba9c47b8cc7fd345ccf104a482265f5ae0ae7
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2412368
Commit-Queue: Robert Ma <robertma@chromium.org>
Reviewed-by: Xianzhu Wang <wangxianzhu@chromium.org>
Auto-Submit: Robert Ma <robertma@chromium.org>
Cr-Commit-Position: refs/heads/master@{#807271}

third_party/blink/tools/blinkpy/web_tests/port/base.py

@@ -135,6 +135,7 @@ class Port(object):
         ('mac10.13', 'x86'),
         ('mac10.14', 'x86'),
         ('mac10.15', 'x86'),
+        ('mac11.0', 'x86'),
         ('win7', 'x86'),
         ('win10', 'x86'),
         ('trusty', 'x86_64'),
@@ -142,10 +143,10 @@ class Port(object):
     )
 
     CONFIGURATION_SPECIFIER_MACROS = {
-        'mac': ['mac10.12', 'mac10.13', 'mac10.14', 'mac10.15'],
+        'mac': ['mac10.12', 'mac10.13', 'mac10.14', 'mac10.15', 'mac11.0'],
         'win': ['win7', 'win10'],
         'linux': ['trusty'],
-        'fuschia': ['fuchsia'],
+        'fuchsia': ['fuchsia'],
     }
 
     # List of ports open on the host that the tests will connect to. When tests

third_party/blink/tools/blinkpy/web_tests/port/base_unittest.py

@@ -39,6 +39,7 @@ from blinkpy.common.system.platform_info_mock import MockPlatformInfo
 from blinkpy.common.system.system_host import SystemHost
 from blinkpy.common.system.system_host_mock import MockSystemHost
 from blinkpy.web_tests.port.base import Port, VirtualTestSuite
+from blinkpy.web_tests.port.factory import PortFactory
 from blinkpy.web_tests.port.test import add_unit_tests_to_mock_filesystem, WEB_TEST_DIR, TestPort
 
 MOCK_WEB_TESTS = '/mock-checkout/' + RELATIVE_WEB_TESTS
@@ -1712,7 +1713,6 @@ class PortTest(LoggingTestCase):
         self.assertIn('--disable-system-font-check',
                       port.additional_driver_flags())
 
-
     def test_enable_tracing(self):
         options, _ = optparse.OptionParser().parse_args([])
         options.enable_tracing = '*,-blink'
@@ -1724,6 +1724,27 @@ class PortTest(LoggingTestCase):
                 '--trace-startup-file=trace_layout_test_non_virtual_TIME.json',
             ], port.args_for_test('non/virtual'))
 
+    def test_all_systems(self):
+        # Port.ALL_SYSTEMS should match CONFIGURATION_SPECIFIER_MACROS.
+        all_systems = []
+        for system in Port.ALL_SYSTEMS:
+            self.assertEqual(len(system), 2)
+            all_systems.append(system[0])
+        all_systems.sort()
+        configuration_specifier_macros = []
+        for macros in Port.CONFIGURATION_SPECIFIER_MACROS.values():
+            configuration_specifier_macros += macros
+        configuration_specifier_macros.sort()
+        self.assertListEqual(all_systems, configuration_specifier_macros)
+
+    def test_configuration_specifier_macros(self):
+        # CONFIGURATION_SPECIFIER_MACROS should contain all SUPPORTED_VERSIONS
+        # of each port. Must use real Port classes in this test.
+        for port_name, versions in Port.CONFIGURATION_SPECIFIER_MACROS.items():
+            port_class, _ = PortFactory.get_port_class(port_name)
+            self.assertIsNotNone(port_class, port_name)
+            self.assertListEqual(versions, list(port_class.SUPPORTED_VERSIONS))
+
 
 class NaturalCompareTest(unittest.TestCase):
     def setUp(self):

third_party/blink/tools/blinkpy/web_tests/port/factory.py

@@ -69,26 +69,35 @@ class PortFactory(object):
 
         _check_configuration_and_target(self._host.filesystem, options)
 
+        port_class, class_name = self.get_port_class(port_name)
+        if port_class is None:
+            raise NotImplementedError('unsupported platform: "%s"' % port_name)
+
+        full_port_name = port_class.determine_full_port_name(
+            self._host, options,
+            class_name if 'browser_test' in port_name else port_name)
+        return port_class(self._host,
+                          full_port_name,
+                          options=options,
+                          **kwargs)
+
+    @classmethod
+    def get_port_class(cls, port_name):
+        """Returns a Port subclass and its name for the given port_name."""
         if 'browser_test' in port_name:
             module_name, class_name = port_name.rsplit('.', 1)
             module = __import__(module_name, globals(), locals(), [], -1)
             port_class_name = module.get_port_class_name(class_name)
             if port_class_name is not None:
-                cls = module.__dict__[port_class_name]
-                port_name = cls.determine_full_port_name(
-                    self._host, options, class_name)
-                return cls(self._host, port_name, options=options, **kwargs)
+                return module.__dict__[port_class_name], class_name
         else:
-            for port_class in self.PORT_CLASSES:
+            for port_class in cls.PORT_CLASSES:
                 module_name, class_name = port_class.rsplit('.', 1)
                 module = __import__(module_name, globals(), locals(), [], -1)
-                cls = module.__dict__[class_name]
-                if port_name.startswith(cls.port_name):
-                    port_name = cls.determine_full_port_name(
-                        self._host, options, port_name)
-                    return cls(
-                        self._host, port_name, options=options, **kwargs)
-        raise NotImplementedError('unsupported platform: "%s"' % port_name)
+                port_class = module.__dict__[class_name]
+                if port_name.startswith(port_class.port_name):
+                    return port_class, class_name
+        return None, None
 
     def all_port_names(self, platform=None):
         """Returns a list of all valid, fully-specified, "real" port names.

third_party/blink/web_tests/TestExpectations

@@ -2531,10 +2531,7 @@ crbug.com/1110512 external/wpt/css/css-pseudo/selection-contenteditable-011.html
 crbug.com/1105958 external/wpt/payment-request/payment-is-showing.https.html [ Timeout ]
 
 # ====== New tests from wpt-importer added here ======
-crbug.com/626703 [ Linux ] external/wpt/fetch/connection-pool/network-partition-key.html [ Timeout ]
-crbug.com/626703 [ Mac ] external/wpt/fetch/connection-pool/network-partition-key.html [ Timeout ]
-crbug.com/626703 [ Mac11.0 ] external/wpt/fetch/connection-pool/network-partition-key.html [ Timeout ]
-crbug.com/626703 [ Win ] external/wpt/fetch/connection-pool/network-partition-key.html [ Timeout ]
+crbug.com/626703 external/wpt/fetch/connection-pool/network-partition-key.html [ Timeout ]
 crbug.com/626703 [ Mac10.15 ] external/wpt/html/browsers/the-window-object/apis-for-creating-and-navigating-browsing-contexts-by-name/open-features-tokenization-top-left.html [ Timeout ]
 crbug.com/626703 external/wpt/css/css-shapes/shape-outside/supported-shapes/circle/shape-outside-circle-043.html [ Failure ]
 crbug.com/626703 external/wpt/css/css-shapes/shape-outside/supported-shapes/ellipse/shape-outside-ellipse-048.html [ Failure ]
@@ -2570,11 +2567,7 @@ crbug.com/626703 external/wpt/html/webappapis/dynamic-markup-insertion/opening-t
 crbug.com/626703 [ Win7 ] external/wpt/html/cross-origin-embedder-policy/reporting-subresource-corp.https.html [ Failure Timeout ]
 crbug.com/626703 external/wpt/css/css-sizing/aspect-ratio/replaced-element-016.tentative.html [ Failure ]
 crbug.com/626703 external/wpt/css/css-sizing/aspect-ratio/replaced-element-021.tentative.html [ Failure ]
-crbug.com/626703 [ Mac10.12 ] external/wpt/webrtc-insertable-streams/RTCPeerConnection-insertable-streams.https.html [ Timeout ]
-crbug.com/626703 [ Mac10.13 ] external/wpt/webrtc-insertable-streams/RTCPeerConnection-insertable-streams.https.html [ Timeout ]
-crbug.com/626703 [ Mac10.14 ] external/wpt/webrtc-insertable-streams/RTCPeerConnection-insertable-streams.https.html [ Timeout ]
-crbug.com/626703 [ Mac10.15 ] external/wpt/webrtc-insertable-streams/RTCPeerConnection-insertable-streams.https.html [ Timeout ]
-crbug.com/626703 [ Mac11.0 ] external/wpt/webrtc-insertable-streams/RTCPeerConnection-insertable-streams.https.html [ Timeout ]
+crbug.com/626703 [ Mac ] external/wpt/webrtc-insertable-streams/RTCPeerConnection-insertable-streams.https.html [ Timeout ]
 crbug.com/626703 external/wpt/content-security-policy/frame-src/frame-src-same-document.sub.html [ Timeout ]
 crbug.com/626703 external/wpt/input-events/input-events-get-target-ranges-backspace.tentative.html [ Timeout ]
 crbug.com/626703 external/wpt/input-events/input-events-get-target-ranges-forwarddelete.tentative.html [ Timeout ]