Commit e52df5fc authored by Mike Jackson's avatar Mike Jackson Committed by Commit Bot

Reland Update axe-core to v3.3.2

This change updates the version of axe-core to the latest released
version. As part of picking up the latest version, the following
additional changes were made:

1) axe-core removed the 'href-no-hash' rule, so that been removed
   from the test list.

2) Several 'link-in-text-block' tests started failing with
   axe-core v3.0.2 (and later). These tests do pass in v3.0.1, but
   are disabled as part of this change.
     - CrManagementA11yTest.SimpleTest_link_in_text_block
     - CrManagementA11yTestWithExtension.
                                ExtensionSection_link_in_text_block
     - CrostiniAccessibilityTest.CROSTINI_link_in_text_block
     - CrExtensionsA11yTest.NoExtensions_link_in_text_block
     - MultideviceA11yTest.MULTIDEVICE_link_in_text_block
     - MultideviceFeaturesA11yTest.
              MULTIDEVICE_FEATURES_ACCESSIBILITY_link_in_text_block

3) One 'heading-order' test started failing in v3.0.0 (and later).
   This tests does pass in V2.4.2, but is disabled as part of this
   change.
     - CrExtensionsA11yTestWithMultipleExensions.
                                       WithExtensions_heading_order

4) One color-contrast test started failing. This test has been disabled
   since there other tests also disabled for this rule
     - WelcomeA11y.WelcomeFlow_color_contrast

5) Several DevTools a11y tests were failing. This change either
   updates the baseline or excludes the rule depending upon the
   test/rule itself.
     - http/tests/devtools/a11y-axe-core/basic-a11y-test.js
     - http/tests/devtools/a11y-axe-core/console-a11y-test.js
     - http/tests/devtools/a11y-axe-core/memory/heap-profiler-a11y-test.js
     - http/tests/devtools/a11y-axe-core/performance/performance-pane-a11y-test.js
     - http/tests/devtools/a11y-axe-core/sources/dom-breakpoints-pane-a11y-test.js

I spent some time investigating the test failures, and it looks
like axe-core is highlighting legitimate product issues.

Bug: 984817
Change-Id: I245af601c4f086d96768fa5677e253095fc53087
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/1841834
Commit-Queue: Mike Jackson <mjackson@microsoft.com>
Reviewed-by: default avatarYang Guo <yangguo@chromium.org>
Reviewed-by: default avatarLorne Mitchell <lomitch@microsoft.com>
Reviewed-by: default avatarHector Carmona <hcarmona@chromium.org>
Cr-Commit-Position: refs/heads/master@{#704624}
parent 86722b36
......@@ -52,7 +52,6 @@ AccessibilityTest.ruleIds = [
'frame-title',
'heading-order',
'hidden-content',
'href-no-hash',
'html-has-lang',
'html-lang-valid',
'image-alt',
......
......@@ -56,6 +56,16 @@ var CrExtensionsA11yTest = class extends PolymerTest {
return parentNode && parentNode.host &&
parentNode.host.tagName == 'CR-TOGGLE';
},
// TODO(crbug.com/1002620): this filter can be removed after
// addressing the bug
'heading-order': function(nodeResult) {
// Filter out 'Heading levels do not increase by one' error when
// enumerating extensions
const expectedMarkup = '<div id="name" role="heading" aria-level="3" \
class="clippable-flex-text">My extension 1</div>';
return nodeResult['html'] === expectedMarkup;
},
};
}
......@@ -85,7 +95,13 @@ AccessibilityTest.define('CrExtensionsA11yTest', {
name: 'NoExtensions',
/** @override */
axeOptions: CrExtensionsA11yTest.axeOptions,
// TODO(crbug.com/1002627): when bug is addressed, this should be replaced
// with axeOptions: CrExtensionsA11yTest.axeOptions,
axeOptions: Object.assign({}, CrExtensionsA11yTest.axeOptions, {
'rules': Object.assign({}, CrExtensionsA11yTest.axeOptions.rules, {
'link-in-text-block': {enabled: false},
})
}),
/** @override */
violationFilter: CrExtensionsA11yTest.violationFilter,
......
......@@ -31,6 +31,8 @@ var CrManagementA11yTest = class extends PolymerTest {
'skip-link': {enabled: false},
// TODO(crbug.com/761461): enable after addressing flaky tests.
'color-contrast': {enabled: false},
// TODO(crbug.com/1002623): remove this line after addressing bug
'link-in-text-block': {enabled: false},
},
};
}
......
......@@ -15,7 +15,7 @@ AccessibilityTest.define('CrostiniAccessibilityTest', {
/** @override */
name: 'CROSTINI',
/** @override */
axeOptions: SettingsAccessibilityTest.axeOptions,
axeOptions: SettingsAccessibilityTest.axeOptionsExcludeLinkInTextBlock,
/** @override */
setup: function() {
settings.router.navigateTo(settings.routes.CROSTINI);
......
......@@ -32,7 +32,7 @@ AccessibilityTest.define('MultideviceA11yTest', {
/** @override */
name: 'MULTIDEVICE',
/** @override */
axeOptions: SettingsAccessibilityTest.axeOptions,
axeOptions: SettingsAccessibilityTest.axeOptionsExcludeLinkInTextBlock,
/** @override */
setup: function() {
settings.router.navigateTo(settings.routes.MULTIDEVICE);
......
......@@ -32,7 +32,7 @@ AccessibilityTest.define('MultideviceFeaturesA11yTest', {
/** @override */
name: 'MULTIDEVICE_FEATURES_ACCESSIBILITY',
/** @override */
axeOptions: SettingsAccessibilityTest.axeOptions,
axeOptions: SettingsAccessibilityTest.axeOptionsExcludeLinkInTextBlock,
/** @override */
setup: function() {
settings.router.navigateTo(settings.routes.MULTIDEVICE_FEATURES);
......
......@@ -28,6 +28,17 @@ SettingsAccessibilityTest.axeOptions = {
}
};
// TODO(crbug.com/1002627): This block prevents generation of a
// link-in-text-block browser-test. This can be removed once the bug is
// addressed, and usage should be replaced with
// SettingsAccessibilityTest.axeOptions
SettingsAccessibilityTest.axeOptionsExcludeLinkInTextBlock =
Object.assign({}, SettingsAccessibilityTest.axeOptions, {
'rules': Object.assign({}, SettingsAccessibilityTest.axeOptions.rules, {
'link-in-text-block': {enabled: false},
})
});
// Default accessibility audit options. Specify in test definition to use.
SettingsAccessibilityTest.violationFilter = {
// Polymer components use aria-active-attribute.
......
......@@ -34,7 +34,12 @@ AccessibilityTest.define('WelcomeA11y', {
name: 'WelcomeFlow',
// Optional. Configuration for axe-core. Can be used to disable a test.
axeOptions: {},
axeOptions: {
'rules': {
// TODO(crbug.com/761461): enable after addressing flaky tests.
'color-contrast': {enabled: false},
}
},
// Optional. Filter on failures. Use this for individual false positives.
violationFilter: {},
......
Name: AXE-CORE Accessibility Audit
Short Name: axe-core
URL: https://github.com/dequelabs/axe-core/
Version: 0
Date: Mon Jul 24 12:17:05 2017
Revision: d02dba3223fefe525438330e40b5da5de81eeeb5
Version: 3.3.2
Date: Thur Aug 22 12:45:00 2019
Revision: 281653df3794f429b71327fe3afa37ca0fadb1c7
License: MPL 2.0
License File: LICENSE
Security Critical: no
......
// Type definitions for axe-core 2.3.1
// Type definitions for axe-core
// Project: https://github.com/dequelabs/axe-core
// Definitions by: Marcy Sutton <https://github.com/marcysutton>
declare module axe {
declare namespace axe {
type ImpactValue = 'minor' | 'moderate' | 'serious' | 'critical';
type ImpactValue = "minor" | "moderate" | "serious" | "critical";
type TagValue = 'wcag2a' | 'wcag2aa' | 'section508' | 'best-practice';
type TagValue = "wcag2a" | "wcag2aa" | "section508" | "best-practice";
type ReporterVersion = 'v1' | 'v2' | 'raw' | 'raw-env' | 'no-passes';
type ReporterVersion = "v1" | "v2";
type RunOnlyType = 'rule' | 'rules' | 'tag' | 'tags';
type RunOnlyType = "rule" | "rules" | "tag" | "tags";
type resultGroups = 'inapplicable' | 'passes' | 'incomplete' | 'violations';
interface ElementContext {
node?: Object,
selector?: string,
include?: any[],
exclude?: any[]
type RunOnlyObject = {
include?: string[] | string[][];
exclude?: string[] | string[][];
};
type RunCallback = (error: Error, results: AxeResults) => void;
type ElementContext = Node | string | RunOnlyObject;
interface TestEngine {
name: string;
version: string;
}
interface TestRunner {
name: string;
}
interface TestEnvironment {
userAgent: string;
windowWidth: number;
windowHeight: number;
orientationAngle?: number;
orientationType?: string;
}
interface RunOnly {
type: RunOnlyType,
value?: {
include?: string[],
exclude?: string[]
}
values?: TagValue[]
type: RunOnlyType;
values?: TagValue[] | string[] | RunOnlyObject;
}
interface RunOptions {
runOnly?: RunOnly;
rules?: Object;
iframes?: boolean;
elementRef?: boolean;
selectors?: boolean;
resultTypes?: resultGroups[];
reporter?: ReporterVersion;
xpath?: boolean;
absolutePaths?: boolean;
restoreScroll?: boolean;
frameWaitTime?: number;
preload?: boolean;
performanceTimer?: boolean;
}
interface AxeResults {
url: string,
timestamp: string,
passes: Result[],
violations: Result[],
incomplete: Result[],
inapplicable: Result[]
toolOptions: RunOptions;
testEngine: TestEngine;
testRunner: TestRunner;
testEnvironment: TestEnvironment;
url: string;
timestamp: string;
passes: Result[];
violations: Result[];
incomplete: Result[];
inapplicable: Result[];
}
interface Result {
description: string,
help: string,
helpUrl: string,
id: string,
impact: ImpactValue,
tags: TagValue[],
nodes: NodeResult[]
description: string;
help: string;
helpUrl: string;
id: string;
impact?: ImpactValue;
tags: TagValue[];
nodes: NodeResult[];
}
interface NodeResult {
html: string,
impact: ImpactValue,
target: string[],
any: CheckResult[],
all: CheckResult[],
none: CheckResult[],
failureSummary?: string
html: string;
impact?: ImpactValue;
target: string[];
xpath?: string[];
any: CheckResult[];
all: CheckResult[];
none: CheckResult[];
failureSummary?: string;
}
interface CheckResult {
id: string,
impact: string,
message: string,
data: any,
relatedNodes?: RelatedNode[]
id: string;
impact: string;
message: string;
data: any;
relatedNodes?: RelatedNode[];
}
interface RelatedNode {
target: string[],
html: string
target: string[];
html: string;
}
interface RuleLocale {
[key: string]: {
description: string;
help: string;
};
}
interface CheckLocale {
[key: string]: {
pass: string;
fail: string;
incomplete: string | { [key: string]: string };
};
}
interface Locale {
lang?: string;
rules?: RuleLocale;
checks?: CheckLocale;
}
interface Spec {
branding?: {
brand: string,
application: string
},
reporter?: ReporterVersion,
checks?: Check[],
rules?: Rule[]
brand?: string;
application?: string;
};
reporter?: ReporterVersion;
checks?: Check[];
rules?: Rule[];
locale?: Locale;
}
interface Check {
id: string,
evaluate: Function,
after?: Function,
options?: any,
matches?: string,
enabled?: boolean
id: string;
evaluate: Function | string;
after?: Function | string;
options?: any;
matches?: string;
enabled?: boolean;
}
interface Rule {
id: string,
selector?: string,
excludeHidden?: boolean,
enabled?: boolean,
pageLevel?: boolean,
any?: string[],
all?: string[],
none?: string[],
tags?: string[],
matches?: string
id: string;
selector?: string;
excludeHidden?: boolean;
enabled?: boolean;
pageLevel?: boolean;
any?: string[];
all?: string[];
none?: string[];
tags?: string[];
matches?: string;
}
interface AxePlugin {
id: string,
run(...args:any[]): any,
id: string;
run(...args: any[]): any;
commands: {
id: string,
callback(...args:any[]): void
}[],
cleanup?(callback:Function): void
id: string;
callback(...args: any[]): void;
}[];
cleanup?(callback: Function): void;
}
let plugins: any
let plugins: any;
/**
* Source string to use as an injected script in Selenium
*/
let source: string
let source: string;
/**
* Object for aXe Results
* Object for axe Results
*/
var AxeResults: AxeResults
var AxeResults: AxeResults;
/**
* Runs a number of rules against the provided HTML page and returns the resulting issue list
*
* @param {Object} context Optional The `Context` specification object @see Context
* @param {Array} options Optional Options passed into rules or checks, temporarily modifying them.
* @param {Function} callback Optional The function to invoke when analysis is complete.
* @returns {any} results If the callback was not defined, aXe will return a Promise instead.
* @param {ElementContext} context Optional The `Context` specification object @see Context
* @param {RunOptions} options Optional Options passed into rules or checks, temporarily modifying them.
* @param {RunCallback} callback Optional The function to invoke when analysis is complete.
* @returns {Promise<AxeResults>|void} If the callback was not defined, axe will return a Promise.
*/
function run(context?: ElementContext, options?: {runOnly?: RunOnly, rules?: Object, iframes?: Boolean, elementRef?: Boolean, selectors?: Boolean}, callback?: (error: Error, results:AxeResults) => void): any
function run(context?: ElementContext): Promise<AxeResults>;
function run(options: RunOptions): Promise<AxeResults>;
function run(callback: (error: Error, results: AxeResults) => void): void;
function run(context: ElementContext, callback: RunCallback): void;
function run(options: RunOptions, callback: RunCallback): void;
function run(
context: ElementContext,
options: RunOptions
): Promise<AxeResults>;
function run(
context: ElementContext,
options: RunOptions,
callback: RunCallback
): void;
/**
* Starts analysis on the current document and its subframes
*
* @param {Object} context The `Context` specification object @see Context
* @param {Array} options Options passed into rules or checks, temporarily modifyint them.
* @param {Function} callback The function to invoke when analysis is complete.
* @returns {Object} results The aXe results object
*/
function a11yCheck(context: ElementContext, options: {runOnly?: RunOnly, rules?: Object, iframes?: Boolean, elementRef?: Boolean, selectors?: Boolean}, callback: (results:AxeResults) => void): AxeResults
/**
* Method for configuring the data format used by aXe. Helpful for adding new
* Method for configuring the data format used by axe. Helpful for adding new
* rules, which must be registered with the library to execute.
* @param {Spec} Spec Object with valid `branding`, `reporter`, `checks` and `rules` data
*/
function configure(spec: Spec): void
function configure(spec: Spec): void;
/**
* Searches and returns rules that contain a tag in the list of tags.
* @param {Array} tags Optional array of tags
* @return {Array} Array of rules
*/
function getRules(tags?: string[]): Object[]
function getRules(tags?: string[]): Object[];
/**
* Restores the default axe configuration
*/
function reset(): void
function reset(): void;
/**
* Function to register a plugin configuration in document and its subframes
* @param {Object} plugin A plugin configuration object
*/
function registerPlugin(plugin: AxePlugin): void
function registerPlugin(plugin: AxePlugin): void;
/**
* Function to clean up plugin configuration in document and its subframes
*/
function cleanup(): void
function cleanup(): void;
}
export = axe;
This diff is collapsed.
This diff is collapsed.
......@@ -14,6 +14,10 @@
'sensors',
];
//TODO(crbug.com/1004940): exclude scrollable-region-focusable for performance.monitor only
const NO_SCROLLABLE_REGION_FOCUSABLE_RULESET = {
'scrollable-region-focusable': { enabled: false, },
};
for (const location of locationsToTest)
await loadViewAndTestElementViolations(location);
......@@ -24,6 +28,7 @@
TestRunner.addResult(`Tests accessibility in the ${view} view using the axe-core linter.`);
await UI.viewManager.showView(view);
const widget = await UI.viewManager.view(view).widget();
await AxeCoreTestRunner.runValidation(widget.element);
const ruleset = view === 'performance.monitor' ? NO_SCROLLABLE_REGION_FOCUSABLE_RULESET : {};
await AxeCoreTestRunner.runValidation(widget.element, ruleset);
}
})();
Tests accessibility in the console using the axe-core linter.
aXe violations: []
aXe violations: [
{
"ruleDescription": "Ensures every ARIA input field has an accessible name",
"helpUrl": "https://dequeuniversity.com/rules/axe/3.3/aria-input-field-name?application=axeAPI",
"ruleId": "aria-input-field-name",
"impact": "serious",
"failedNodes": [
{
"target": [
[
".console-main-toolbar",
".toolbar-input-prompt"
]
],
"html": "<div class=\"toolbar-input-prompt text-prompt\" role=\"textbox\" contenteditable=\"plaintext-only\" data-placeholder=\"Filter\" aria-placeholder=\"Filter\"></div>",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute or the title attribute is empty"
}
]
}
]
......@@ -10,6 +10,7 @@
await UI.viewManager.showView('console');
const widget = await UI.viewManager.view('console').widget();
//TODO(crbug.com/1004940): expected.txt file has 'aria-input-field-name' exceptions
await AxeCoreTestRunner.runValidation(widget.element);
TestRunner.completeTest();
})();
Tests accessibility in heap profiler using the axe-core linter.
aXe violations: []
aXe violations: [
{
"ruleDescription": "Ensures role attribute has an appropriate value for the element",
"helpUrl": "https://dequeuniversity.com/rules/axe/3.3/aria-allowed-role?application=axeAPI",
"ruleId": "aria-allowed-role",
"impact": "minor",
"failedNodes": [
{
"target": [
"form"
],
"html": "<form role=\"radiogroup\" aria-label=\"Select profiling type\">",
"failureSummary": "Fix any of the following:\n ARIA role radiogroup is not allowed for given element"
}
]
}
]
......@@ -37,6 +37,7 @@
await UI.viewManager.showView('heap_profiler');
const widget = await UI.viewManager.view('heap_profiler').widget();
//TODO(crbug.com/1004940): expected.txt file has 'aria-allowed-role' exceptions
await AxeCoreTestRunner.runValidation(widget.element);
TestRunner.completeTest();
})();
......@@ -6,11 +6,49 @@ aXe violations: []
Running: testBottomUpView
Tests accessibility in performance BottomUp view using the axe-core linter
aXe violations: []
aXe violations: [
{
"ruleDescription": "Ensures every ARIA input field has an accessible name",
"helpUrl": "https://dequeuniversity.com/rules/axe/3.3/aria-input-field-name?application=axeAPI",
"ruleId": "aria-input-field-name",
"impact": "serious",
"failedNodes": [
{
"target": [
[
".widget.vbox[slot=\"insertion-point-main\"] > .toolbar",
".toolbar-input-prompt"
]
],
"html": "<div class=\"toolbar-input-prompt text-prompt\" role=\"textbox\" contenteditable=\"plaintext-only\" data-placeholder=\"Filter\" aria-placeholder=\"Filter bottom-up\"></div>",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute or the title attribute is empty"
}
]
}
]
Running: testCallTreeView
Tests accessibility in performance CallTree view using the axe-core linter
aXe violations: []
aXe violations: [
{
"ruleDescription": "Ensures every ARIA input field has an accessible name",
"helpUrl": "https://dequeuniversity.com/rules/axe/3.3/aria-input-field-name?application=axeAPI",
"ruleId": "aria-input-field-name",
"impact": "serious",
"failedNodes": [
{
"target": [
[
".widget.vbox[slot=\"insertion-point-main\"] > .toolbar",
"div[aria-placeholder=\"Filter\\ call\\ tree\"]"
]
],
"html": "<div class=\"toolbar-input-prompt text-prompt\" role=\"textbox\" contenteditable=\"plaintext-only\" data-placeholder=\"Filter\" aria-placeholder=\"Filter call tree\"></div>",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Element has no title attribute or the title attribute is empty"
}
]
}
]
......@@ -66,6 +66,7 @@
model.timelineModel().minimumRecordTime(),
model.timelineModel().maximumRecordTime()));
//TODO(crbug.com/1004940): expected.txt file has 'aria-input-field-name' exceptions
await AxeCoreTestRunner.runValidation(detailsTab.element);
}
......
......@@ -6,31 +6,31 @@ Running the axe-core linter on the DOM breakpoints pane.
aXe violations: [
{
"ruleDescription": "Ensures every form element has a label",
"helpUrl": "https://dequeuniversity.com/rules/axe/3.0/label?application=axeAPI",
"helpUrl": "https://dequeuniversity.com/rules/axe/3.3/label?application=axeAPI",
"ruleId": "label",
"impact": "critical",
"failedNodes": [
{
"target": [
[
"div:nth-child(7) > div.vbox.flex-auto",
".widget.vbox > .breakpoint-list > div:nth-child(1) > span",
".flex-none.flex-auto.vbox:nth-child(7) > .flex-auto.vbox",
".breakpoint-entry:nth-child(1) > span[is=\"dt-checkbox\"]",
"#ui-checkbox-label9"
]
],
"html": "<input type=\"checkbox\" id=\"ui-checkbox-label9\" aria-label=\"Subtree modified: div#rootElement\">",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty or not visible\n Form element does not have an implicit (wrapped) <label>\n Form element does not have an explicit <label>\n Element has no title attribute or the title attribute is empty"
"html": "<input type=\"checkbox\" id=\"ui-checkbox-label9\">",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Form element does not have an implicit (wrapped) <label>\n Form element does not have an explicit <label>\n Element has no title attribute or the title attribute is empty"
},
{
"target": [
[
"div:nth-child(7) > div.vbox.flex-auto",
".widget.vbox > .breakpoint-list > div:nth-child(2) > span",
".flex-none.flex-auto.vbox:nth-child(7) > .flex-auto.vbox",
".breakpoint-entry:nth-child(2) > span[is=\"dt-checkbox\"]",
"#ui-checkbox-label10"
]
],
"html": "<input type=\"checkbox\" id=\"ui-checkbox-label10\" aria-label=\"Node removed: div#hostElement\">",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty or not visible\n Form element does not have an implicit (wrapped) <label>\n Form element does not have an explicit <label>\n Element has no title attribute or the title attribute is empty"
"html": "<input type=\"checkbox\" id=\"ui-checkbox-label10\">",
"failureSummary": "Fix any of the following:\n aria-label attribute does not exist or is empty\n aria-labelledby attribute does not exist, references elements that do not exist or references elements that are empty\n Form element does not have an implicit (wrapped) <label>\n Form element does not have an explicit <label>\n Element has no title attribute or the title attribute is empty"
}
]
}
......
......@@ -36,6 +36,7 @@
TestRunner.addResult(
'Running the axe-core linter on the DOM breakpoints pane.');
//TODO(crbug.com/1004940): expected.txt file has 'label' exceptions
await AxeCoreTestRunner.runValidation(domBreakpointContainer.element);
TestRunner.completeTest();
})();
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment