🤠 Disable flaky payment chrome_public_test_apk on Marshmallow.

The following tests were flaky on Marshmallow the last ~10 build:

org.chromium.chrome.browser.payments
 ├──ExpandablePaymentHandlerChangePaymentMethodTest
 │   ├──testDetails
 │   ├──testDetailsBasicCard
 │   ├──testReject
 │   └──testThrow
 ├──ExpandablePaymentHandlerChangePaymentMethodTest
 │   └──testThrowBasicCard
 ├──ExpandablePaymentHandlerTest
 │   ├──testNavigateBackWithSystemBackButton
 │   ├──testSecureConnectionShowUi__CERT_AUTO
 │   ├──testSecureConnectionShowUi__CERT_COMMON_NAME_IS_DOMAIN
 │   └──testSecureConnectionShowUi__CERT_OK_BY_INTERMEDIATE
 ├──PaymentHandlerChangePaymentMethodTest
 │   ├──testDetails
 │   ├──testDetailsBasicCard
 │   ├──testNoEventHandlerBasicCard
 │   ├──testReject
 │   ├──testRejectBasicCard
 │   └──testThrow
 ├──PaymentHandlerEnableDelegationsTest
 │   ├──testPartialDelegationContactInfoNotSupported
 │   ├──testShippingAndContactInfoDelegation
 │   └──testShippingDelegation
 ├──PaymentRequestPaymentMethodIdentifierTest
 │   └──testAddMasterCard
 └──PaymentRequestRemoveBillingAddressTest
     └──testPayWithCard

TBR=rouslan@chromium.org

Bug: 1102320
Change-Id: I182afa734c763abb5d377c5fde8a2510e85b6794
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2283162Reviewed-by: Arthur Sonzogni <arthursonzogni@chromium.org>
Commit-Queue: Arthur Sonzogni <arthursonzogni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#785335}

chrome/android/javatests/src/org/chromium/chrome/browser/payments/ExpandablePaymentHandlerChangePaymentMethodTest.java

@@ -4,6 +4,7 @@
 
 package org.chromium.chrome.browser.payments;
 
+import android.os.Build.VERSION_CODES;
 import android.support.test.InstrumentationRegistry;
 
 import androidx.test.filters.MediumTest;
@@ -16,6 +17,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.chromium.base.test.util.CommandLineFlags;
+import org.chromium.base.test.util.DisableIf;
 import org.chromium.base.test.util.Feature;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.flags.ChromeSwitches;
@@ -100,7 +102,11 @@ public class ExpandablePaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testReject() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testReject() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testReject", mRule.getDismissed());
         mRule.expectResultContains(
@@ -128,7 +134,11 @@ public class ExpandablePaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testThrow() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testThrow() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testThrow", mRule.getDismissed());
         mRule.expectResultContains(
@@ -141,7 +151,11 @@ public class ExpandablePaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testThrowBasicCard() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testThrowBasicCard() throws Throwable {
         mRule.clickNode("basicCardMethodName");
         installPaymentHandler();
         mRule.triggerUIAndWait("testThrow", mRule.getReadyToPay());
@@ -156,7 +170,11 @@ public class ExpandablePaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testDetails() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testDetails() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testDetails", mRule.getDismissed());
         // Look for the this exact return value to ensure that the browser redacts some details
@@ -179,7 +197,11 @@ public class ExpandablePaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testDetailsBasicCard() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testDetailsBasicCard() throws Throwable {
         mRule.clickNode("basicCardMethodName");
         installPaymentHandler();
         mRule.triggerUIAndWait("testDetails", mRule.getReadyToPay());

chrome/android/javatests/src/org/chromium/chrome/browser/payments/ExpandablePaymentHandlerTest.java

@@ -15,6 +15,7 @@ import static androidx.test.espresso.matcher.ViewMatchers.withId;
 import static androidx.test.espresso.matcher.ViewMatchers.withText;
 
 import android.os.Build;
+import android.os.Build.VERSION_CODES;
 import android.support.test.InstrumentationRegistry;
 import android.support.test.uiautomator.UiDevice;
 
@@ -369,6 +370,9 @@ public class ExpandablePaymentHandlerTest {
             sdk_is_greater_than = Build.VERSION_CODES.LOLLIPOP_MR1,
             sdk_is_less_than = Build.VERSION_CODES.N)
     @Feature({"Payments"})
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
     public void
     testNavigateBackWithSystemBackButton() throws Throwable {
         startDefaultServer();
@@ -419,7 +423,11 @@ public class ExpandablePaymentHandlerTest {
     @SmallTest
     @Feature({"Payments"})
     @ParameterAnnotations.UseMethodParameter(GoodCertParams.class)
-    public void testSecureConnectionShowUi(int goodCertificate) throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testSecureConnectionShowUi(int goodCertificate) throws Throwable {
         startServer(goodCertificate);
         PaymentHandlerCoordinator paymentHandler = createPaymentHandlerAndShow(mDefaultIsIncognito);
         waitForTitleShown(paymentHandler.getWebContentsForTest());

chrome/android/javatests/src/org/chromium/chrome/browser/payments/PaymentHandlerChangePaymentMethodTest.java

@@ -4,6 +4,7 @@
 
 package org.chromium.chrome.browser.payments;
 
+import android.os.Build.VERSION_CODES;
 import android.support.test.InstrumentationRegistry;
 
 import androidx.test.filters.MediumTest;
@@ -16,6 +17,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.chromium.base.test.util.CommandLineFlags;
+import org.chromium.base.test.util.DisableIf;
 import org.chromium.base.test.util.Feature;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.flags.ChromeSwitches;
@@ -72,7 +74,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testNoEventHandler() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testNoEventHandler() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testNoHandler", mRule.getDismissed());
         mRule.expectResultContains(
@@ -85,7 +91,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testNoEventHandlerBasicCard() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testNoEventHandlerBasicCard() throws Throwable {
         mRule.clickNode("basicCardMethodName");
         installPaymentHandler();
         mRule.triggerUIAndWait("testNoHandler", mRule.getReadyToPay());
@@ -100,7 +110,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testReject() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testReject() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testReject", mRule.getDismissed());
         mRule.expectResultContains(
@@ -113,7 +127,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testRejectBasicCard() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testRejectBasicCard() throws Throwable {
         mRule.clickNode("basicCardMethodName");
         installPaymentHandler();
         mRule.triggerUIAndWait("testReject", mRule.getReadyToPay());
@@ -128,7 +146,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testThrow() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testThrow() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testThrow", mRule.getDismissed());
         mRule.expectResultContains(
@@ -156,7 +178,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testDetails() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testDetails() throws Throwable {
         installPaymentHandler();
         mRule.clickNodeAndWait("testDetails", mRule.getDismissed());
         // Look for the this exact return value to ensure that the browser redacts some details
@@ -179,7 +205,11 @@ public class PaymentHandlerChangePaymentMethodTest {
      */
     @Test
     @Feature({"Payments"})
-    public void testDetailsBasicCard() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testDetailsBasicCard() throws Throwable {
         mRule.clickNode("basicCardMethodName");
         installPaymentHandler();
         mRule.triggerUIAndWait("testDetails", mRule.getReadyToPay());

chrome/android/javatests/src/org/chromium/chrome/browser/payments/PaymentHandlerEnableDelegationsTest.java

@@ -4,6 +4,7 @@
 
 package org.chromium.chrome.browser.payments;
 
+import android.os.Build.VERSION_CODES;
 import android.support.test.InstrumentationRegistry;
 import android.view.View;
 
@@ -19,6 +20,7 @@ import org.junit.runner.RunWith;
 
 import org.chromium.base.test.util.CallbackHelper;
 import org.chromium.base.test.util.CommandLineFlags;
+import org.chromium.base.test.util.DisableIf;
 import org.chromium.base.test.util.Feature;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.flags.ChromeSwitches;
@@ -86,7 +88,11 @@ public class PaymentHandlerEnableDelegationsTest {
     @Test
     @Feature({"Payments"})
     @MediumTest
-    public void testShippingDelegation() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testShippingDelegation() throws Throwable {
         installPaymentHandlerWithDelegations("['shippingAddress']");
         // The pay button should be enabled when shipping address is requested and the selected
         // payment instrument can provide it.
@@ -114,7 +120,11 @@ public class PaymentHandlerEnableDelegationsTest {
     @Test
     @Feature({"Payments"})
     @MediumTest
-    public void testShippingAndContactInfoDelegation() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testShippingAndContactInfoDelegation() throws Throwable {
         installPaymentHandlerWithDelegations(
                 "['shippingAddress', 'payerName', 'payerEmail', 'payerPhone']");
         // The pay button should be enabled when shipping address and payer's contact information
@@ -145,7 +155,11 @@ public class PaymentHandlerEnableDelegationsTest {
     @Test
     @Feature({"Payments"})
     @MediumTest
-    public void testPartialDelegationContactInfoNotSupported() throws Throwable {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testPartialDelegationContactInfoNotSupported() throws Throwable {
         installPaymentHandlerWithDelegations("['shippingAddress']");
         createPaymentRequestAndWaitFor(
                 "{requestShipping: true, requestPayerName: true, requestPayerEmail: true}",

chrome/android/javatests/src/org/chromium/chrome/browser/payments/PaymentRequestPaymentMethodIdentifierTest.java

@@ -8,6 +8,8 @@ import static org.chromium.chrome.browser.payments.PaymentRequestTestRule.DECEMB
 import static org.chromium.chrome.browser.payments.PaymentRequestTestRule.FIRST_BILLING_ADDRESS;
 import static org.chromium.chrome.browser.payments.PaymentRequestTestRule.NEXT_YEAR;
 
+import android.os.Build.VERSION_CODES;
+
 import androidx.test.filters.MediumTest;
 
 import org.junit.Before;
@@ -17,6 +19,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.chromium.base.test.util.CommandLineFlags;
+import org.chromium.base.test.util.DisableIf;
 import org.chromium.base.test.util.Feature;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.autofill.AutofillTestHelper;
@@ -111,7 +114,11 @@ public class PaymentRequestPaymentMethodIdentifierTest implements MainActivitySt
     @Test
     @MediumTest
     @Feature({"Payments"})
-    public void testPayWithBasicCard() throws TimeoutException {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testPayWithBasicCard() throws TimeoutException {
         mPaymentRequestTestRule.openPageAndClickNodeAndWait(
                 "checkBasicVisa", mPaymentRequestTestRule.getCanMakePaymentQueryResponded());
         mPaymentRequestTestRule.expectResultContains(new String[] {"true"});
@@ -135,7 +142,11 @@ public class PaymentRequestPaymentMethodIdentifierTest implements MainActivitySt
     @Test
     @MediumTest
     @Feature({"Payments"})
-    public void testAddMasterCard() throws TimeoutException {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testAddMasterCard() throws TimeoutException {
         mPaymentRequestTestRule.triggerUIAndWait(mPaymentRequestTestRule.getReadyToPay());
         mPaymentRequestTestRule.clickInPaymentMethodAndWait(
                 R.id.payments_section, mPaymentRequestTestRule.getReadyForInput());

chrome/android/javatests/src/org/chromium/chrome/browser/payments/PaymentRequestRemoveBillingAddressTest.java

@@ -8,6 +8,8 @@ import static org.chromium.chrome.browser.payments.PaymentRequestTestRule.DECEMB
 import static org.chromium.chrome.browser.payments.PaymentRequestTestRule.FIRST_BILLING_ADDRESS;
 import static org.chromium.chrome.browser.payments.PaymentRequestTestRule.NEXT_YEAR;
 
+import android.os.Build.VERSION_CODES;
+
 import androidx.test.filters.MediumTest;
 
 import org.junit.ClassRule;
@@ -16,6 +18,7 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.chromium.base.test.util.CommandLineFlags;
+import org.chromium.base.test.util.DisableIf;
 import org.chromium.base.test.util.Feature;
 import org.chromium.chrome.R;
 import org.chromium.chrome.browser.autofill.AutofillTestHelper;
@@ -65,7 +68,11 @@ public class PaymentRequestRemoveBillingAddressTest implements MainActivityStart
     @Test
     @MediumTest
     @Feature({"Payments"})
-    public void testPayWithCard() throws TimeoutException {
+    @DisableIf.
+    Build(sdk_is_greater_than = VERSION_CODES.LOLLIPOP_MR1, sdk_is_less_than = VERSION_CODES.N,
+            message = "Flaky on Marshmallow https://crbug.com/1102320")
+    public void
+    testPayWithCard() throws TimeoutException {
         mPaymentRequestTestRule.triggerUIAndWait(mPaymentRequestTestRule.getReadyForInput());
 
         // Expand the payment section.