[webauthn] WPT: Clean up virtual authenticators

Add a step on web platform tests to clean up any virtual authenticators
that are created. Tests should be self contained and clean up after
themselves.

Fixed: 1056231
Change-Id: I2fc3f892efbc602d5bee8ec7cbf3d51873cd6522
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2078832
Auto-Submit: Nina Satragno <nsatragno@chromium.org>
Reviewed-by: Robert Ma <robertma@chromium.org>
Commit-Queue: Nina Satragno <nsatragno@chromium.org>
Cr-Commit-Position: refs/heads/master@{#745134}

third_party/blink/web_tests/external/wpt/webauthn/createcredential-badargs-authnrselection.https-expected.txt

 This is a testharness.js-based test.
-FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection is empty array assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "NotSupportedError: The user agent does not implement a password store." ("NotSupportedError") expected object "TypeError" ("TypeError")
-FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection is null assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "NotSupportedError: The user agent does not implement a password store." ("NotSupportedError") expected object "TypeError" ("TypeError")
+PASS Set up the test environment
+FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection is empty array assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
+FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection is empty string
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection is string
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection attachment is empty string
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection attachment is empty object
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection attachment is null
-FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection attachment platform assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "NotSupportedError: The user agent does not implement a password store." that is not a DOMException NotAllowedError: property "code" is equal to 9, expected 0
-FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection residentKey true assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "NotSupportedError: The user agent does not implement a password store." that is not a DOMException NotAllowedError: property "code" is equal to 9, expected 0
-FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection residentKey is string assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "NotSupportedError: The user agent does not implement a password store." ("NotSupportedError") expected object "TypeError" ("TypeError")
+PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection attachment platform
+PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection residentKey true
+FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection residentKey is string promise_rejects_js: Expected bad parameters to fail function "function() { throw e }" threw object "NotAllowedError: The operation either timed out or was not allowed. See: https://w3c.github.io/webauthn/#sec-assertion-privacy." ("NotAllowedError") expected instance of function "function TypeError() { [native code] }" ("TypeError")
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection userVerification empty string
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection userVerification empty object
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection userVerification bad value
 PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection userVerification null
-FAIL Bad AuthenticatorSelectionCriteria: authenticatorSelection userVerification required assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "NotSupportedError: The user agent does not implement a password store." that is not a DOMException NotAllowedError: property "code" is equal to 9, expected 0
+PASS Bad AuthenticatorSelectionCriteria: authenticatorSelection userVerification required
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/createcredential-badargs-rp.https-expected.txt

 This is a testharness.js-based test.
+PASS Set up the test environment
 PASS Bad rp: rp missing
 PASS Bad rp: rp null
 PASS Bad rp: rp is string
 PASS Bad rp: rp is empty object
-FAIL Bad rp: id is object assert_throws: Expected bad parameters to fail function "function() { throw e }" threw object "SecurityError: The relying party ID '[object Object]' is not a registrable domain suffix of, nor equal to 'https://web-platform.test:8444'." ("SecurityError") expected object "TypeError" ("TypeError")
+FAIL Bad rp: id is object promise_rejects_js: Expected bad parameters to fail function "function() { throw e }" threw object "SecurityError: The relying party ID is not a registrable domain suffix of, nor equal to the current domain." ("SecurityError") expected instance of function "function TypeError() { [native code] }" ("TypeError")
 PASS Bad rp: id is null
 PASS Bad rp: id is empty String
 PASS Bad rp: id is invalid domain (has space)
@@ -13,9 +14,10 @@ PASS rp missing name
 FAIL Bad rp: name is object assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 FAIL Bad rp: name is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 FAIL Bad rp: name is empty String assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad rp: icon is object assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad rp: icon is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
+FAIL Bad rp: icon is object promise_rejects_js: Expected bad parameters to fail function "function() { throw e }" threw object "SecurityError: 'rp.icon' should be a secure URL" ("SecurityError") expected instance of function "function TypeError() { [native code] }" ("TypeError")
+FAIL Bad rp: icon is null promise_rejects_js: Expected bad parameters to fail function "function() { throw e }" threw object "SecurityError: 'rp.icon' should be a secure URL" ("SecurityError") expected instance of function "function TypeError() { [native code] }" ("TypeError")
 FAIL Bad rp: icon is empty String assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 PASS Bad rp: icon is insecure
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/createcredential-badargs-user.https-expected.txt

-This is a testharness.js-based test.
-PASS Bad user: user missing
-PASS Bad user: user is string
-PASS Bad user: user is empty object
-PASS Bad user: id is undefined
-PASS Bad user: id is object
-PASS Bad user: id is null
-PASS Bad user: id is empty String
-PASS Bad user: id is empty Array
-FAIL Bad user: id is empty ArrayBuffer assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-PASS Bad user: ArrayBuffer id is too long (65 bytes)
-PASS Bad user: Int16Array id is too long (66 bytes)
-PASS Bad user: Int32Array id is too long (68 bytes)
-PASS Bad user: Float32Array id is too long (68 bytes)
-PASS Bad user: Float64Array id is too long (72 bytes)
-PASS Bad user: id is too long (65 bytes)
-PASS user missing name
-FAIL Bad user: name is object assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: name is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: name is empty String assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: icon is object assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: icon is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: icon is empty String assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-PASS Bad user: icon is insecure String
-PASS Bad user: displayName is undefined
-FAIL Bad user: displayName is object assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: displayName is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-FAIL Bad user: displayName is empty String assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
-Harness: the test ran to completion.
-

third_party/blink/web_tests/external/wpt/webauthn/createcredential-excludecredentials.https-expected.txt

 This is a testharness.js-based test.
+PASS Set up the test environment
 PASS Bad excludeCredentials: string
 PASS Bad excludeCredentials: empty object
 PASS excludeCredentials missing
 PASS excludeCredentials empty array
-FAIL exclude existing credential assert_throws: expected to fail on excluded credenetial function "function() { throw e }" threw object "InvalidStateError: The user attempted to register an authenticator that contains one of the credentials already registered with the relying party." that is not a DOMException NotAllowedError: property "code" is equal to 11, expected 0
+FAIL exclude existing credential promise_rejects_dom: expected to fail on excluded credenetial function "function() { throw e }" threw object "InvalidStateError: The user attempted to register an authenticator that contains one of the credentials already registered with the relying party." that is not a DOMException NotAllowedError: property "code" is equal to 11, expected 0
 PASS exclude random (non-existing) credential
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/createcredential-extensions.https-expected.txt

 This is a testharness.js-based test.
+PASS Set up the test environment
 PASS Bad extensions: extensions is string
 FAIL Bad extensions: extensions is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 FAIL Bad extensions: extensions is empty Array assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
@@ -10,5 +11,6 @@ PASS extensions is a nonsensical JSON string
 PASS empty appid in create request
 PASS null appid in create request
 PASS appid in create request
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/createcredential-passing.https-expected.txt

 This is a testharness.js-based test.
+PASS Set up the test environment
 PASS passing credentials.create() with default arguments
-FAIL passing credentials.create() with rpId (host and port) promise_test: Unhandled rejection with value: object "SecurityError: The relying party ID 'web-platform.test:8444' is not a registrable domain suffix of, nor equal to 'https://web-platform.test:8444'."
+FAIL passing credentials.create() with rpId (host and port) promise_test: Unhandled rejection with value: object "SecurityError: The relying party ID is not a registrable domain suffix of, nor equal to the current domain."
 PASS passing credentials.create() with rpId (hostname)
 PASS passing credentials.create() without rp.icon
 PASS very short user id
@@ -37,5 +38,6 @@ PASS attestation parameter: attestation is undefined
 PASS extensions undefined
 PASS extensions are empty object
 PASS extensions are dict of empty strings
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/createcredential-pubkeycredparams.https-expected.txt

+This is a testharness.js-based test.
+PASS Set up the test environment
+PASS Bad pubKeyCredParams: pubKeyCredParams is undefined
+PASS Bad pubKeyCredParams: pubKeyCredParams is string
+PASS Bad pubKeyCredParams: pubKeyCredParams is null
+PASS Bad pubKeyCredParams: first param has bad type ("something-else")
+PASS Bad pubKeyCredParams: first param has bad type ("")
+PASS Bad pubKeyCredParams: first param has bad type (null)
+PASS Bad pubKeyCredParams: first param has bad type (empty object)
+FAIL Bad pubKeyCredParams: first param has bad alg (42) promise_rejects_dom: Expected  bad parameters to fail function "function() { throw e }" threw object "NotAllowedError: The operation either timed out or was not allowed. See: https://w3c.github.io/webauthn/#sec-assertion-privacy." that is not a DOMException NotSupportedError: property "code" is equal to 0, expected 9
+FAIL Bad pubKeyCredParams: first param has bad alg (0) promise_rejects_dom: Expected  bad parameters to fail function "function() { throw e }" threw object "NotAllowedError: The operation either timed out or was not allowed. See: https://w3c.github.io/webauthn/#sec-assertion-privacy." that is not a DOMException NotSupportedError: property "code" is equal to 0, expected 9
+PASS Clean up the test environment
+Harness: the test ran to completion.
+

third_party/blink/web_tests/external/wpt/webauthn/createcredential-timeout.https.html

@@ -22,11 +22,12 @@
 promise_test(async t => {
     // if available, configure a mock authenticator that does not respond to user input
     try {
-        await window.test_driver.add_virtual_authenticator({
+        let authenticator = await window.test_driver.add_virtual_authenticator({
             protocol: "ctap1/u2f",
             transport: "usb",
             isUserConsenting: false,
         });
+      t.add_cleanup(() => window.test_driver.remove_virtual_authenticator(authenticator));
     } catch (error) {
         if (error !== "error: Action add_virtual_authenticator not implemented") {
             throw error;

third_party/blink/web_tests/external/wpt/webauthn/getcredential-badargs-userverification.https-expected.txt

-This is a testharness.js-based test.
-FAIL Bad userVerification: empty string promise_test: Unhandled rejection with value: object "Error: NotSupportedError: The user agent does not implement a password store."
-FAIL Bad userVerification: empty object promise_test: Unhandled rejection with value: object "Error: NotSupportedError: The user agent does not implement a password store."
-FAIL Bad userVerification: bad value promise_test: Unhandled rejection with value: object "Error: NotSupportedError: The user agent does not implement a password store."
-FAIL Bad userVerification: null promise_test: Unhandled rejection with value: object "Error: NotSupportedError: The user agent does not implement a password store."
-FAIL Bad userVerification: "required" promise_test: Unhandled rejection with value: object "Error: NotSupportedError: The user agent does not implement a password store."
-Harness: the test ran to completion.
-

third_party/blink/web_tests/external/wpt/webauthn/getcredential-extensions.https-expected.txt

 This is a testharness.js-based test.
+PASS Set up the test environment
 PASS Bad extensions: extensions is string
 FAIL Bad extensions: extensions is null assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 FAIL Bad extensions: extensions is empty Array assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
@@ -7,5 +8,6 @@ FAIL Bad extensions: malformatted JSON assert_unreached: Should have rejected: E
 FAIL Bad extensions: JavaScript object assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 FAIL Bad extensions: extension ID too long assert_unreached: Should have rejected: Expected bad parameters to fail Reached unreachable code
 PASS extensions is a nonsensical JSON string
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/getcredential-passing.https-expected.txt

 This is a testharness.js-based test.
+PASS Set up the test environment
 PASS passing credentials.get() with default args
 PASS passing credentials.create() with no timeout
 PASS rpId undefined
-FAIL passing credentials.get() with rpId (host and port) promise_test: Unhandled rejection with value: object "SecurityError: The relying party ID 'web-platform.test:8444' is not a registrable domain suffix of, nor equal to 'https://web-platform.test:8444'."
+FAIL passing credentials.get() with rpId (host and port) promise_test: Unhandled rejection with value: object "SecurityError: The relying party ID is not a registrable domain suffix of, nor equal to the current domain."
 PASS passing credentials.get() with rpId (hostname)
 FAIL no credential specified promise_test: Unhandled rejection with value: object "Error: Attempting list without defining credential to test"
 PASS authenticatorSelection userVerification undefined
@@ -11,5 +12,6 @@ PASS authenticatorSelection userVerification discouraged
 PASS extensions undefined
 PASS extensions are empty object
 PASS extensions are dict of empty strings
+PASS Clean up the test environment
 Harness: the test ran to completion.
 

third_party/blink/web_tests/external/wpt/webauthn/getcredential-timeout.https-expected.txt

+This is a testharness.js-based test.
+FAIL WebAuthn navigator.credentials.get() timeout Tests assert_unreached: Should have rejected: undefined Reached unreachable code
+Harness: the test ran to completion.
+

third_party/blink/web_tests/external/wpt/webauthn/getcredential-timeout.https.html

@@ -22,6 +22,7 @@ promise_test(async t => {
             transport: "usb",
             isUserConsenting: false,
         });
+        t.add_cleanup(() => window.test_driver.remove_virtual_authenticator(authenticator));
         const private_key =
             "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg8_zMDQDYAxlU-Q"
           + "hk1Dwkf0v18GZca1DMF3SaJ9HPdmShRANCAASNYX5lyVCOZLzFZzrIKmeZ2jwU"

third_party/blink/web_tests/external/wpt/webauthn/helpers.js

@@ -537,17 +537,28 @@ function validateAuthenticatorAssertionResponse(assert) {
 
 function standardSetup(cb) {
     // Setup an automated testing environment if available.
-    window.test_driver.add_virtual_authenticator({
-      protocol: "ctap1/u2f",
-      transport: "usb"
-    }).then(cb).catch(error => {
-        if (error === "error: Action add_virtual_authenticator not implemented") {
-          // The protocol is not available. Continue manually.
-          cb();
-          return;
+    let authenticator;
+    promise_test(async t => {
+        try {
+            authenticator = await window.test_driver.add_virtual_authenticator({
+                protocol: "ctap1/u2f",
+                transport: "usb"
+            });
+        } catch (error) {
+            if (error !== "error: Action add_virtual_authenticator not implemented") {
+                throw error;
+            }
+            // The protocol is not available. Continue manually.
         }
-        throw error;
-    });
+    }, "Set up the test environment");
+
+    cb();
+
+    promise_test(t => {
+        if (authenticator) {
+            return window.test_driver.remove_virtual_authenticator(authenticator);
+        }
+    }, "Clean up the test environment");
 }
 
 /* JSHINT */

third_party/blink/web_tests/external/wpt/webauthn/webauthn-testdriver-basic.https.html

@@ -10,8 +10,10 @@
 <script>
 "use strict";
 
+let authenticator;
+
 promise_test(async t => {
-  return window.test_driver.add_virtual_authenticator({
+  authenticator = await window.test_driver.add_virtual_authenticator({
     protocol: "ctap1/u2f",
     transport: "usb",
   });
@@ -114,4 +116,8 @@ promise_test(async t => {
     { name: "ECDSA", hash: "SHA-256" }, key, signature, signedData));
 }, "Get an assertion");
 
+promise_test(async t => {
+  await window.test_driver.remove_virtual_authenticator(authenticator);
+}, "Clean up the test environment");
+
 </script>