feat: test a11y with Cypress (#11199)

* feat: test a11y with Cypress

* axe test more pages

* archive a11y violations on success too

* remove date from file path

* don't warn if no accessibility files to upload... they're not on all test jobs

.github/workflows/docker-image-test.yml

@@ -100,7 +100,7 @@ jobs:
 
     cypress:
         name: Cypress E2E tests (${{ strategy.job-index }})
-        if: ${{ github.ref != 'refs/heads/master' }} # Don't run on master, we only cace about node_modules cache
+        if: ${{ github.ref != 'refs/heads/master' }} # Don't run on master, we only care about node_modules cache
         runs-on: ubuntu-latest
         timeout-minutes: 30
         needs: [build, cypress_prep]
@@ -189,6 +189,13 @@ jobs:
                   path: cypress/videos
               if: ${{ failure() }}
 
+            - name: Archive accessibility violations
+              uses: actions/upload-artifact@v3
+              with:
+                  name: accessibility-violations
+                  path: '**/a11y/'
+                  if-no-files-found: 'ignore'
+
             - name: Show logs on failure
               # use artefact here, as I think the output will be too large for display in an action
               uses: actions/upload-artifact@v3

.gitignore

@@ -41,4 +41,5 @@ ee/benchmarks/results
 coverage-*.xml
 object_storage/
 __emails__
-share/GeoLite2-City.mmdb
+share/GeoLite2-City.mmdb
+a11y/

bin/e2e-test-runner

@@ -71,6 +71,8 @@ export DATABASE_URL="postgres://${PGUSER}:${PGPASSWORD}@${PGHOST}:${PGPORT}/${DA
 trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
 
 recreateDatabases() {
+  echo "~~~~~~~~~~~~~ dropping and creating databases ~~~~~~~~~~~~~"
+
   dropdb --if-exists $DATABASE
   createdb $DATABASE
 
@@ -80,11 +82,13 @@ recreateDatabases() {
 }
 
 migrateDatabases() {
+  echo "~~~~~~~~~~~~~ migrating databases ~~~~~~~~~~~~~"
   python manage.py migrate
   python manage.py migrate_clickhouse
 }
 
 setupDev() {
+  echo "~~~~~~~~~~~~~ setting up django ~~~~~~~~~~~~~"
   python manage.py setup_dev &
 }
 

cypress/e2e/a11y.js

@@ -0,0 +1,33 @@
+import { reportA11y } from '../support/accessibility'
+
+describe('a11y', () => {
+    it('home should have no accessibility violations', () => {
+        cy.get('[data-attr="menu-item-projecthomepage"]').click()
+        cy.injectAxe()
+        reportA11y({ includedImpacts: ['serious', 'critical'] }, 'home-page')
+    })
+
+    const sidebarItems = [
+        'dashboards',
+        'savedinsights',
+        'sessionsrecordings',
+        'featureflags',
+        'experiments',
+        'events',
+        'datamanagement',
+        'persons',
+        'cohorts',
+        'annotations',
+        'plugins',
+        'toolbarlaunch',
+        'projectsettings',
+    ]
+
+    sidebarItems.forEach((sideBarItem) => {
+        it(`${sideBarItem} should have no accessibility violations`, () => {
+            cy.clickNavMenu(sideBarItem)
+            cy.injectAxe()
+            reportA11y({ includedImpacts: ['serious', 'critical'] }, sideBarItem)
+        })
+    })
+})

cypress/support/accessibility.js

@@ -0,0 +1,17 @@
+export const reportA11y = (options, tag) => {
+    if (typeof tag !== undefined) {
+        tag += '-'
+    }
+
+    // reports on A11y failures without failing the tests
+    cy.checkA11y(
+        null,
+        options,
+        (violations) => {
+            cy.log(`${violations.length} violation(s) detected`)
+
+            cy.writeFile(`a11y/${tag}accessibility-violations.json`, JSON.stringify(violations))
+        },
+        true
+    )
+}

cypress/support/e2e.js

@@ -1,5 +1,6 @@
 import 'givens/setup'
 import './commands'
+import 'cypress-axe'
 import { decideResponse } from 'cypress/fixtures/api/decide'
 
 try {

package.json

@@ -153,6 +153,7 @@
         "@typescript-eslint/eslint-plugin": "^3.6.0",
         "@typescript-eslint/parser": "^3.6.0",
         "autoprefixer": "^10.4.7",
+        "axe-core": "^4.4.3",
         "babel-eslint": "^10.1.0",
         "babel-jest": "^27.0.6",
         "babel-loader": "^8.0.6",
@@ -162,6 +163,7 @@
         "css-loader": "^3.4.2",
         "cssnano": "^4.1.10",
         "cypress": "^10.4.0",
+        "cypress-axe": "^1.0.0",
         "cypress-terminal-report": "^4.1.1",
         "esbuild": "^0.14.42",
         "esbuild-plugin-less": "^1.1.7",

yarn.lock

@@ -5779,6 +5779,11 @@ aws4@^1.8.0:
   resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59"
   integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA==
 
+axe-core@^4.4.3:
+  version "4.4.3"
+  resolved "https://registry.yarnpkg.com/axe-core/-/axe-core-4.4.3.tgz#11c74d23d5013c0fa5d183796729bc3482bd2f6f"
+  integrity sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==
+
 babel-eslint@^10.1.0:
   version "10.1.0"
   resolved "https://registry.yarnpkg.com/babel-eslint/-/babel-eslint-10.1.0.tgz#6968e568a910b78fb3779cdd8b6ac2f479943232"
@@ -7613,6 +7618,11 @@ cyclist@^1.0.1:
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
   integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=
 
+cypress-axe@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/cypress-axe/-/cypress-axe-1.0.0.tgz#ab4e9486eaa3bb956a90a1ae40d52df42827b4f0"
+  integrity sha512-QBlNMAd5eZoyhG8RGGR/pLtpHGkvgWXm2tkP68scJ+AjYiNNOlJihxoEwH93RT+rWOLrefw4iWwEx8kpEcrvJA==
+
 cypress-terminal-report@^4.1.1:
   version "4.1.1"
   resolved "https://registry.yarnpkg.com/cypress-terminal-report/-/cypress-terminal-report-4.1.1.tgz#245d731ca2db3d23e0ac1307073d57574606c496"