From 8e339732757e0499c120497a3f920234068b1f15 Mon Sep 17 00:00:00 2001 From: Ralph Cowling Date: Thu, 28 Jul 2022 12:36:52 +0200 Subject: [PATCH 1/4] Add an array of severity levels to fail on --- package.json | 148 +++++++++++++++++++++++++-------------------------- src/index.ts | 23 +++++--- 2 files changed, 91 insertions(+), 80 deletions(-) diff --git a/package.json b/package.json index 30f279f..08dcaa8 100644 --- a/package.json +++ b/package.json @@ -1,76 +1,76 @@ { - "name": "cypress-axe", - "version": "0.8.1", - "license": "MIT", - "description": "Test accessibility with axe-core in Cypress", - "homepage": "https://github.com/component-driven/cypress-axe", - "repository": "component-driven/cypress-axe", - "files": [ - "dist" - ], - "main": "dist/index.js", - "types": "dist/index.d.ts", - "scripts": { - "build": "tsc", - "lint": "eslint . --cache --fix", - "pretest": "npm run lint", - "test": "npm run test:e2e:ci", - "posttest": "npm run format", - "format": "prettier --loglevel warn --write \"**/*.{js,md}\"", - "prepublishOnly": "npm run build", - "start": "http-server test", - "cypress": "cypress open", - "cypress:headless": "cypress run --browser chrome --headless", - "test:e2e": "start-server-and-test start 8080 cypress", - "test:e2e:ci": "start-server-and-test start 8080 cypress:headless" - }, - "engines": { - "node": ">=10" - }, - "dependencies": {}, - "peerDependencies": { - "axe-core": "^3 || ^4", - "cypress": "^10" - }, - "devDependencies": { - "@types/node": "^14.14.8", - "@typescript-eslint/eslint-plugin": "^4.8.1", - "@typescript-eslint/parser": "^4.8.1", - "axe-core": "^4.0.2", - "cypress": "^10.1.0", - "eslint": "^7.12.0", - "eslint-config-tamia": "^7.2.6", - "http-server": "^0.12.3", - "husky": "^4.3.0", - "lint-staged": "^10.5.0", - "prettier": "^2.1.2", - "start-server-and-test": "^1.11.5", - "typescript": "^4.0.5" - }, - "authors": [ - { - "name": "Andy Van Slaars", - "url": "https://vanslaars.io" - }, - { - "name": "Artem Sapegin", - "url": "https://sapegin.me" - } - ], - "keywords": [ - "a11y", - "accessibility", - "axe", - "axe-core", - "cypress" - ], - "husky": { - "hooks": { - "pre-commit": "lint-staged" - } - }, - "lint-staged": { - "*.js": "eslint --cache --fix", - "*.{js,md}": "prettier --write" - } + "name": "cypress-axe", + "version": "1.0.1-rc", + "license": "MIT", + "description": "Test accessibility with axe-core in Cypress", + "homepage": "https://github.com/component-driven/cypress-axe", + "repository": "component-driven/cypress-axe", + "files": [ + "dist" + ], + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "lint": "eslint . --cache --fix", + "pretest": "npm run lint", + "test": "npm run test:e2e:ci", + "posttest": "npm run format", + "format": "prettier --loglevel warn --write \"**/*.{js,md}\"", + "prepublishOnly": "npm run build", + "start": "http-server test", + "cypress": "cypress open", + "cypress:headless": "cypress run --browser chrome --headless", + "test:e2e": "start-server-and-test start 8080 cypress", + "test:e2e:ci": "start-server-and-test start 8080 cypress:headless" + }, + "engines": { + "node": ">=10" + }, + "dependencies": {}, + "peerDependencies": { + "axe-core": "^3 || ^4", + "cypress": "^10" + }, + "devDependencies": { + "@types/node": "^14.14.8", + "@typescript-eslint/eslint-plugin": "^4.8.1", + "@typescript-eslint/parser": "^4.8.1", + "axe-core": "^4.0.2", + "cypress": "^10.1.0", + "eslint": "^7.12.0", + "eslint-config-tamia": "^7.2.6", + "http-server": "^0.12.3", + "husky": "^4.3.0", + "lint-staged": "^10.5.0", + "prettier": "^2.1.2", + "start-server-and-test": "^1.11.5", + "typescript": "^4.0.5" + }, + "authors": [ + { + "name": "Andy Van Slaars", + "url": "https://vanslaars.io" + }, + { + "name": "Artem Sapegin", + "url": "https://sapegin.me" + } + ], + "keywords": [ + "a11y", + "accessibility", + "axe", + "axe-core", + "cypress" + ], + "husky": { + "hooks": { + "pre-commit": "lint-staged" + } + }, + "lint-staged": { + "*.js": "eslint --cache --fix", + "*.{js,md}": "prettier --write" + } } diff --git a/src/index.ts b/src/index.ts index 89d9981..1db102b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,6 +21,10 @@ export interface Options extends axe.RunOptions { includedImpacts?: string[]; } +export type AlertLevels = + | false + | ['minor' | 'moderate' | 'serious' | 'critical']; + export const injectAxe = () => { const fileName = typeof require?.resolve === 'function' @@ -50,7 +54,7 @@ const checkA11y = ( context?: axe.ElementContext, options?: Options, violationCallback?: (violations: axe.Result[]) => void, - skipFailures = false + failOn: AlertLevels = false ) => { cy.window({ log: false }) .then((win) => { @@ -100,13 +104,20 @@ const checkA11y = ( return cy.wrap(violations, { log: false }); }) .then((violations) => { - if (!skipFailures) { + const violated = + failOn && + Array.isArray(failOn) && + Boolean(failOn.length) && + violations.filter((v) => v.impact && failOn.includes(v.impact)); + + if (violated) { assert.equal( - violations.length, + violated.length, 0, - `${violations.length} accessibility violation${ - violations.length === 1 ? '' : 's' - } ${violations.length === 1 ? 'was' : 'were'} detected` + `Failure set on errors type ${failOn} + ${violated.length} accessibility violation${violated.length === 1 ? '' : 's'} ${ + violated.length === 1 ? 'was' : 'were' + } detected` ); } else if (violations.length) { Cypress.log({ From b199dc279a3d586c752a7c09c90cd4e63c265f79 Mon Sep 17 00:00:00 2001 From: Ralph Cowling Date: Thu, 28 Jul 2022 15:38:12 +0200 Subject: [PATCH 2/4] allow various levels of failure: all, none, or SeverityLevels --- src/index.ts | 57 ++++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/src/index.ts b/src/index.ts index 1db102b..1e4c63a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,9 +21,9 @@ export interface Options extends axe.RunOptions { includedImpacts?: string[]; } -export type AlertLevels = - | false - | ['minor' | 'moderate' | 'serious' | 'critical']; +export type SeverityLevels = ['minor' | 'moderate' | 'serious' | 'critical']; + +export type AlertLevels = 'any' | 'none' | SeverityLevels; export const injectAxe = () => { const fileName = @@ -50,11 +50,23 @@ function isEmptyObjectorNull(value: any) { return Object.entries(value).length === 0 && value.constructor === Object; } +const assertOnViolations = (violations: any[], failLevel: AlertLevels) => { + assert.equal( + violations.length, + 0, + ` + == ${failLevel ? 'Failure set on errors of type ' + failLevel : ''} == + ${violations.length} accessibility violation${ + violations.length === 1 ? '' : 's' + } ${violations.length === 1 ? 'was' : 'were'} detected` + ); +}; + const checkA11y = ( context?: axe.ElementContext, options?: Options, violationCallback?: (violations: axe.Result[]) => void, - failOn: AlertLevels = false + failOn: AlertLevels = 'any' ) => { cy.window({ log: false }) .then((win) => { @@ -104,28 +116,21 @@ const checkA11y = ( return cy.wrap(violations, { log: false }); }) .then((violations) => { - const violated = - failOn && - Array.isArray(failOn) && - Boolean(failOn.length) && - violations.filter((v) => v.impact && failOn.includes(v.impact)); - - if (violated) { - assert.equal( - violated.length, - 0, - `Failure set on errors type ${failOn} - ${violated.length} accessibility violation${violated.length === 1 ? '' : 's'} ${ - violated.length === 1 ? 'was' : 'were' - } detected` - ); - } else if (violations.length) { - Cypress.log({ - name: 'a11y violation summary', - message: `${violations.length} accessibility violation${ - violations.length === 1 ? '' : 's' - } ${violations.length === 1 ? 'was' : 'were'} detected`, - }); + if (failOn === 'none') { + return; + } + if (failOn === 'any') { + assertOnViolations(violations, failOn); + } else { + const violated = + failOn && + Array.isArray(failOn) && + Boolean(failOn.length) && + violations.filter((v) => v.impact && failOn.includes(v.impact)); + + if (violated) { + assertOnViolations(violated, failOn); + } } }); }; From 88255e1085fb33f45afdaa0ae79fa30315581bd9 Mon Sep 17 00:00:00 2001 From: Ralph Cowling Date: Thu, 28 Jul 2022 16:25:06 +0200 Subject: [PATCH 3/4] explain failure levels --- README.md | 13 +++-- package.json | 148 +++++++++++++++++++++++++-------------------------- 2 files changed, 83 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index 6282a3f..3601b47 100644 --- a/README.md +++ b/README.md @@ -131,11 +131,11 @@ Allows you to define a callback that receives the violations for custom side-eff **NOTE:** _This respects the `includedImpacts` filter and will only execute with violations that are included._ -##### skipFailures (optional, defaults to false) +##### failOn (optional, defaults to 'any') -Disables assertions based on violations and only logs violations to the console output. This enabled you to see violations while allowing your tests to pass. This should be used as a temporary measure while you address accessibility violations. +Set a level to fail on. Options are 'any', 'none' or an array of levels ['minor', 'moderate', 'serious', 'critical']; -Reference : https://github.com/component-driven/cypress-axe/issues/17 +Allows you to set the level of severity to fail on. If set to 'any', any level of violation will fail, whilst 'none' will allow all violations to pass. You can fine tune this with a severity array, e.g. `['serious', 'critical']` will only fail on errors at the serious or critical level. This allows you to log all violations using e.g. `cy.checkA11y(null, null, terminalLog)` whilst only failing for a particular severity level. ### Examples @@ -175,7 +175,12 @@ it('Has no a11y violations after button click', () => { it('Only logs a11y violations while allowing the test to pass', () => { // Do not fail the test when there are accessibility failures - cy.checkA11y(null, null, null, true) + cy.checkA11y(null, null, null, 'none') +}) + +it('Only logs a11y violations of level "critical"', () => { + // Do not fail the test when there are accessibility failures below the 'critical' level + cy.checkA11y(null, null, ['critical']) }) ``` diff --git a/package.json b/package.json index 08dcaa8..30f279f 100644 --- a/package.json +++ b/package.json @@ -1,76 +1,76 @@ { - "name": "cypress-axe", - "version": "1.0.1-rc", - "license": "MIT", - "description": "Test accessibility with axe-core in Cypress", - "homepage": "https://github.com/component-driven/cypress-axe", - "repository": "component-driven/cypress-axe", - "files": [ - "dist" - ], - "main": "dist/index.js", - "types": "dist/index.d.ts", - "scripts": { - "build": "tsc", - "lint": "eslint . --cache --fix", - "pretest": "npm run lint", - "test": "npm run test:e2e:ci", - "posttest": "npm run format", - "format": "prettier --loglevel warn --write \"**/*.{js,md}\"", - "prepublishOnly": "npm run build", - "start": "http-server test", - "cypress": "cypress open", - "cypress:headless": "cypress run --browser chrome --headless", - "test:e2e": "start-server-and-test start 8080 cypress", - "test:e2e:ci": "start-server-and-test start 8080 cypress:headless" - }, - "engines": { - "node": ">=10" - }, - "dependencies": {}, - "peerDependencies": { - "axe-core": "^3 || ^4", - "cypress": "^10" - }, - "devDependencies": { - "@types/node": "^14.14.8", - "@typescript-eslint/eslint-plugin": "^4.8.1", - "@typescript-eslint/parser": "^4.8.1", - "axe-core": "^4.0.2", - "cypress": "^10.1.0", - "eslint": "^7.12.0", - "eslint-config-tamia": "^7.2.6", - "http-server": "^0.12.3", - "husky": "^4.3.0", - "lint-staged": "^10.5.0", - "prettier": "^2.1.2", - "start-server-and-test": "^1.11.5", - "typescript": "^4.0.5" - }, - "authors": [ - { - "name": "Andy Van Slaars", - "url": "https://vanslaars.io" - }, - { - "name": "Artem Sapegin", - "url": "https://sapegin.me" - } - ], - "keywords": [ - "a11y", - "accessibility", - "axe", - "axe-core", - "cypress" - ], - "husky": { - "hooks": { - "pre-commit": "lint-staged" - } - }, - "lint-staged": { - "*.js": "eslint --cache --fix", - "*.{js,md}": "prettier --write" - } + "name": "cypress-axe", + "version": "0.8.1", + "license": "MIT", + "description": "Test accessibility with axe-core in Cypress", + "homepage": "https://github.com/component-driven/cypress-axe", + "repository": "component-driven/cypress-axe", + "files": [ + "dist" + ], + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "lint": "eslint . --cache --fix", + "pretest": "npm run lint", + "test": "npm run test:e2e:ci", + "posttest": "npm run format", + "format": "prettier --loglevel warn --write \"**/*.{js,md}\"", + "prepublishOnly": "npm run build", + "start": "http-server test", + "cypress": "cypress open", + "cypress:headless": "cypress run --browser chrome --headless", + "test:e2e": "start-server-and-test start 8080 cypress", + "test:e2e:ci": "start-server-and-test start 8080 cypress:headless" + }, + "engines": { + "node": ">=10" + }, + "dependencies": {}, + "peerDependencies": { + "axe-core": "^3 || ^4", + "cypress": "^10" + }, + "devDependencies": { + "@types/node": "^14.14.8", + "@typescript-eslint/eslint-plugin": "^4.8.1", + "@typescript-eslint/parser": "^4.8.1", + "axe-core": "^4.0.2", + "cypress": "^10.1.0", + "eslint": "^7.12.0", + "eslint-config-tamia": "^7.2.6", + "http-server": "^0.12.3", + "husky": "^4.3.0", + "lint-staged": "^10.5.0", + "prettier": "^2.1.2", + "start-server-and-test": "^1.11.5", + "typescript": "^4.0.5" + }, + "authors": [ + { + "name": "Andy Van Slaars", + "url": "https://vanslaars.io" + }, + { + "name": "Artem Sapegin", + "url": "https://sapegin.me" + } + ], + "keywords": [ + "a11y", + "accessibility", + "axe", + "axe-core", + "cypress" + ], + "husky": { + "hooks": { + "pre-commit": "lint-staged" + } + }, + "lint-staged": { + "*.js": "eslint --cache --fix", + "*.{js,md}": "prettier --write" + } } From 1672a81160e6fff5f56a013ef2ff1cfd5390ec0c Mon Sep 17 00:00:00 2001 From: Ralph Cowling Date: Fri, 29 Jul 2022 11:45:10 +0200 Subject: [PATCH 4/4] update package to fork --- package.json | 148 +++++++++++++++++++++++++-------------------------- 1 file changed, 74 insertions(+), 74 deletions(-) diff --git a/package.json b/package.json index 30f279f..eeb7e7f 100644 --- a/package.json +++ b/package.json @@ -1,76 +1,76 @@ { - "name": "cypress-axe", - "version": "0.8.1", - "license": "MIT", - "description": "Test accessibility with axe-core in Cypress", - "homepage": "https://github.com/component-driven/cypress-axe", - "repository": "component-driven/cypress-axe", - "files": [ - "dist" - ], - "main": "dist/index.js", - "types": "dist/index.d.ts", - "scripts": { - "build": "tsc", - "lint": "eslint . --cache --fix", - "pretest": "npm run lint", - "test": "npm run test:e2e:ci", - "posttest": "npm run format", - "format": "prettier --loglevel warn --write \"**/*.{js,md}\"", - "prepublishOnly": "npm run build", - "start": "http-server test", - "cypress": "cypress open", - "cypress:headless": "cypress run --browser chrome --headless", - "test:e2e": "start-server-and-test start 8080 cypress", - "test:e2e:ci": "start-server-and-test start 8080 cypress:headless" - }, - "engines": { - "node": ">=10" - }, - "dependencies": {}, - "peerDependencies": { - "axe-core": "^3 || ^4", - "cypress": "^10" - }, - "devDependencies": { - "@types/node": "^14.14.8", - "@typescript-eslint/eslint-plugin": "^4.8.1", - "@typescript-eslint/parser": "^4.8.1", - "axe-core": "^4.0.2", - "cypress": "^10.1.0", - "eslint": "^7.12.0", - "eslint-config-tamia": "^7.2.6", - "http-server": "^0.12.3", - "husky": "^4.3.0", - "lint-staged": "^10.5.0", - "prettier": "^2.1.2", - "start-server-and-test": "^1.11.5", - "typescript": "^4.0.5" - }, - "authors": [ - { - "name": "Andy Van Slaars", - "url": "https://vanslaars.io" - }, - { - "name": "Artem Sapegin", - "url": "https://sapegin.me" - } - ], - "keywords": [ - "a11y", - "accessibility", - "axe", - "axe-core", - "cypress" - ], - "husky": { - "hooks": { - "pre-commit": "lint-staged" - } - }, - "lint-staged": { - "*.js": "eslint --cache --fix", - "*.{js,md}": "prettier --write" - } + "name": "@-ralph/cypress-axe", + "version": "1.0.0rc", + "license": "MIT", + "description": "Test accessibility with axe-core in Cypress", + "homepage": "https://github.com/component-driven/cypress-axe", + "repository": "component-driven/cypress-axe", + "files": [ + "dist" + ], + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc", + "lint": "eslint . --cache --fix", + "pretest": "npm run lint", + "test": "npm run test:e2e:ci", + "posttest": "npm run format", + "format": "prettier --loglevel warn --write \"**/*.{js,md}\"", + "prepublishOnly": "npm run build", + "start": "http-server test", + "cypress": "cypress open", + "cypress:headless": "cypress run --browser chrome --headless", + "test:e2e": "start-server-and-test start 8080 cypress", + "test:e2e:ci": "start-server-and-test start 8080 cypress:headless" + }, + "engines": { + "node": ">=10" + }, + "dependencies": {}, + "peerDependencies": { + "axe-core": "^3 || ^4", + "cypress": "^10" + }, + "devDependencies": { + "@types/node": "^14.14.8", + "@typescript-eslint/eslint-plugin": "^4.8.1", + "@typescript-eslint/parser": "^4.8.1", + "axe-core": "^4.0.2", + "cypress": "^10.1.0", + "eslint": "^7.12.0", + "eslint-config-tamia": "^7.2.6", + "http-server": "^0.12.3", + "husky": "^4.3.0", + "lint-staged": "^10.5.0", + "prettier": "^2.1.2", + "start-server-and-test": "^1.11.5", + "typescript": "^4.0.5" + }, + "authors": [ + { + "name": "Andy Van Slaars", + "url": "https://vanslaars.io" + }, + { + "name": "Artem Sapegin", + "url": "https://sapegin.me" + } + ], + "keywords": [ + "a11y", + "accessibility", + "axe", + "axe-core", + "cypress" + ], + "husky": { + "hooks": { + "pre-commit": "lint-staged" + } + }, + "lint-staged": { + "*.js": "eslint --cache --fix", + "*.{js,md}": "prettier --write" + } }