Trial browser tests in CircleCI

.circleci/config.yml

@@ -47,11 +47,66 @@ jobs:
             - node_modules
           key: frontend-v1-{{ checksum "package-lock.json" }}
       - run: npm run dist
+      # Save static files for subsequent jobs.
+      - persist_to_workspace:
+          root: ~/project
+          paths:
+            - wagtail
       - run: npm run lint:js
       - run: npm run lint:css
       - run: npm run test:unit:coverage -- --runInBand
       - run: bash <(curl -s https://codecov.io/bash) -F frontend
 
+  ui_tests:
+    docker:
+      - image: cimg/python:3.8.11-browsers
+    environment:
+      PIPENV_VENV_IN_PROJECT: true
+      DJANGO_SETTINGS_MODULE: wagtail.tests.settings_ui
+    steps:
+      - checkout
+      - attach_workspace:
+          at: ~/project
+      - restore_cache:
+          key: pipenv-v1-{{ checksum "setup.py" }}
+      # Only install if .venv wasn’t cached.
+      - run: |
+            if [[ ! -e ".venv" ]]; then
+                pipenv install -e .[testing]
+            fi
+      - save_cache:
+          key: pipenv-v1-{{ checksum "setup.py" }}
+          paths:
+            - .venv
+      - restore_cache:
+          key: ui_tests-npm_integration-v1-{{ checksum "client/tests/integration/package-lock.json" }}
+      # Only install if node_modules wasn’t cached.
+      - run: |
+            if [[ ! -e "client/tests/integration/node_modules" ]]; then
+                npm --prefix ./client/tests/integration install --no-save --no-optional --no-audit --no-fund --progress=false
+            fi
+      - save_cache:
+          key: ui_tests-npm_integration-v1-{{ checksum "client/tests/integration/package-lock.json" }}
+          paths:
+            - client/tests/integration/node_modules
+      - run: pipenv run ./wagtail/tests/manage.py migrate
+      - run:
+          command: pipenv run ./wagtail/tests/manage.py runserver 0:8000
+          background: true
+      - run: pipenv run ./wagtail/tests/manage.py createcachetable
+      - run:
+          command: pipenv run ./wagtail/tests/manage.py createsuperuser --noinput
+          environment:
+            DJANGO_SUPERUSER_EMAIL: admin@example.com
+            DJANGO_SUPERUSER_USERNAME: admin
+            DJANGO_SUPERUSER_PASSWORD: changeme
+      - run:
+          command: npm run test:integration -- --runInBand --reporters=default --reporters=jest-junit
+          environment:
+            JEST_JUNIT_OUTPUT_DIR: reports/jest
+      - store_test_results:
+          path: ./reports/jest
+
   nightly-build:
     docker:
       - image: cimg/python:3.8.11
@@ -73,6 +128,9 @@ workflows:
       jobs:
         - backend
         - frontend
+        - ui_tests:
+            requires:
+              - frontend
 
     nightly:
       jobs:

.gitignore

@@ -10,7 +10,7 @@
 /.tox/
 /.venv
 /venv
-/node_modules/
+node_modules
 npm-debug.log*
 *.idea/
 /*.egg/
@@ -24,7 +24,8 @@ npm-debug.log*
 *.ipr
 *.iws
 coverage/
-client/node_modules
 
 ### vscode
 .vscode
+
+*.db

client/tests/integration/.eslintrc.js

@@ -0,0 +1,25 @@
+/**
+ * Overrides to our base ESLint configuration specifically for our integration tests.
+ */
+module.exports = {
+  rules: {
+    // To support code running in Node without transpiling.
+    '@typescript-eslint/no-var-requires': 'off',
+    'no-underscore-dangle': ['error', { allow: ['__BROWSER_GLOBAL__'] }],
+    // So we can lint the code without resolving imports, in case the sub-package isn’t installed.
+    'import/no-unresolved': 'off',
+  },
+  env: {
+    jest: true,
+    browser: true,
+    node: true
+  },
+  globals: {
+    page: 'readonly'
+  },
+  settings: {
+    'import/resolver': {
+      webpack: null,
+    }
+  },
+};

client/tests/integration/PuppeteerEnvironment.js

@@ -0,0 +1,45 @@
+const { readFile } = require('fs').promises;
+const os = require('os');
+const path = require('path');
+const puppeteer = require('puppeteer');
+const NodeEnvironment = require('jest-environment-node');
+
+const DIR = path.join(os.tmpdir(), 'jest_puppeteer_global_setup');
+
+/**
+ * Custom Puppeteer environment as documented on https://jestjs.io/docs/puppeteer.
+ * We don’t use jest-puppeteer because it’s unreliable.
+ */
+class PuppeteerEnvironment extends NodeEnvironment {
+  async setup() {
+    await super.setup();
+    // get the wsEndpoint
+    const wsEndpoint = await readFile(path.join(DIR, 'wsEndpoint'), 'utf8');
+    if (!wsEndpoint) {
+      throw new Error('wsEndpoint not found');
+    }
+
+    // connect to puppeteer
+    this.global.browser = await puppeteer.connect({
+      browserWSEndpoint: wsEndpoint,
+      defaultViewport: {
+        width: 1024,
+        height: 768,
+      },
+    });
+
+    this.global.page = await this.global.browser.newPage();
+  }
+
+  async teardown() {
+    await this.global.page.close();
+
+    await super.teardown();
+  }
+
+  getVmContext() {
+    return super.getVmContext();
+  }
+}
+
+module.exports = PuppeteerEnvironment;

client/tests/integration/editbird.test.js

@@ -0,0 +1,16 @@
+describe('Editbird', () => {
+  beforeAll(async () => {
+    await page.goto('http://localhost:8000/');
+  });
+
+  it('axe', async () => {
+    const trigger = await page.$('[aria-controls="wagtail-userbar-items"]');
+    await Promise.all([
+      trigger.click(),
+      page.waitForSelector('[aria-labelledby="wagtail-userbar-trigger"]', { visible: true }),
+    ]);
+    await expect(page).toPassAxeTests({
+      exclude: '[role="menuitem"]'
+    });
+  });
+});

client/tests/integration/editor.test.js

@@ -0,0 +1,76 @@
+describe('Editor', () => {
+  const globalEditorExcludes =
+    '.skiplink, .sidebar__collapse-toggle, #wagtail-sidebar, li[aria-controls^="tab-"]';
+  beforeAll(async () => {
+    await page.goto(
+      'http://localhost:8000/admin/pages/add/demosite/standardpage/2/'
+    );
+  });
+
+  it('has the right heading', async () => {
+    const pageHeader = await page.$('h1');
+    const pageHeaderValue = await pageHeader.evaluate((el) => el.textContent);
+    expect(pageHeaderValue).toContain('New Standard page');
+  });
+
+  it('axe', async () => {
+    await expect(page).toPassAxeTests({
+      exclude: `${globalEditorExcludes}, [aria-describedby^="placeholder-"]`,
+    });
+  });
+
+  it('axe InlinePanel', async () => {
+    const toggle = await page.$('.sidebar__collapse-toggle');
+    toggle.click();
+    const trigger = await page.$('#id_carousel_items-ADD');
+    trigger.click();
+    await expect(page).toPassAxeTests({
+      exclude: `${globalEditorExcludes}, [aria-describedby^="placeholder-"]`,
+    });
+  });
+
+  it('axe embed chooser', async () => {
+    const trigger = await page.$('.Draftail-Editor [name="EMBED"]');
+    await Promise.all([
+      trigger.click(),
+      page.waitForSelector('.embed-form', { visible: true }),
+    ]);
+    await expect(page).toPassAxeTests({
+      exclude: `${globalEditorExcludes}, [aria-describedby^="placeholder-"], .modal`,
+    });
+    await Promise.all([
+      await page.keyboard.press('Escape'),
+      page.waitForSelector('.Draftail-Editor--readonly', { hidden: true }),
+    ]);
+  });
+
+  it('axe image chooser', async () => {
+    const trigger = await page.$('.Draftail-Editor [name="IMAGE"]');
+    await Promise.all([
+      trigger.click(),
+      page.waitForSelector('.image-search', { visible: true }),
+    ]);
+    await expect(page).toPassAxeTests({
+      exclude: `${globalEditorExcludes}, [aria-describedby^="placeholder-"], .modal`,
+    });
+    await Promise.all([
+      await page.keyboard.press('Escape'),
+      page.waitForSelector('.Draftail-Editor--readonly', { hidden: true }),
+    ]);
+  });
+
+  it('axe page chooser', async () => {
+    const trigger = await page.$('.Draftail-Editor [name="LINK"]');
+    await Promise.all([
+      trigger.click(),
+      page.waitForSelector('.page-results', { visible: true }),
+    ]);
+    await expect(page).toPassAxeTests({
+      exclude: `${globalEditorExcludes}, [aria-describedby^="placeholder-"], .modal`,
+    });
+    await Promise.all([
+      await page.keyboard.press('Escape'),
+      page.waitForSelector('.Draftail-Editor--readonly', { hidden: true }),
+    ]);
+  });
+});

client/tests/integration/groups.test.js

@@ -0,0 +1,15 @@
+describe('Groups', () => {
+  beforeAll(async () => {
+    await page.goto('http://localhost:8000/admin/groups/2/');
+  }, 10000);
+
+  it('has the right heading', async () => {
+    expect(await page.title()).toContain('Wagtail - Editing Editors');
+  });
+
+  it('axe', async () => {
+    await expect(page).toPassAxeTests({
+      exclude: '.skiplink, .sidebar__collapse-toggle, #wagtail-sidebar'
+    });
+  });
+});

client/tests/integration/homepage.test.js

@@ -0,0 +1,43 @@
+describe('Homepage', () => {
+  beforeAll(async () => {
+    await page.goto('http://localhost:8000/admin/', {
+      waitUntil: 'domcontentloaded',
+    });
+  });
+
+  it('has the right heading', async () => {
+    const pageHeader = await page.$('h1');
+    const pageHeaderValue = await pageHeader.evaluate((el) => el.textContent);
+    expect(pageHeaderValue).toContain('Welcome to the Test Site Wagtail CMS');
+  });
+
+  it('axe', async () => {
+    await expect(page).toPassAxeTests({
+      exclude: '.stats, .skiplink, #wagtail-sidebar, .sidebar__collapse-toggle',
+    });
+  });
+
+  it('axe page explorer', async () => {
+    const trigger = await page.$('[aria-haspopup="dialog"]');
+    await trigger.click();
+    await expect(page).toPassAxeTests({
+      include: '.sidebar-main-menu',
+    });
+  });
+
+  it('axe sidebar sub-menu', async () => {
+    const trigger = await page.$('[aria-haspopup="true"]');
+    await trigger.click();
+    await expect(page).toPassAxeTests({
+      include: '.sidebar-main-menu',
+    });
+  });
+
+  it('axe sidebar footer', async () => {
+    const trigger = await page.$('[aria-label="Edit your account"]');
+    await trigger.click();
+    await expect(page).toPassAxeTests({
+      include: '.sidebar-footer',
+    });
+  });
+});

client/tests/integration/jest.config.js

@@ -0,0 +1,9 @@
+module.exports = {
+  globalSetup: './setup.js',
+  globalTeardown: './teardown.js',
+  testEnvironment: './PuppeteerEnvironment.js',
+  setupFilesAfterEnv: [
+    'expect-puppeteer',
+    '@wordpress/jest-puppeteer-axe'
+  ]
+};

client/tests/integration/listing.test.js

@@ -0,0 +1,15 @@
+describe('Listing', () => {
+  beforeAll(async () => {
+    await page.goto('http://localhost:8000/admin/pages/2/');
+  });
+
+  it('has the right heading', async () => {
+    expect(await page.title()).toContain('Wagtail - Exploring Welcome to your new Wagtail site!');
+  });
+
+  it('axe', async () => {
+    await expect(page).toPassAxeTests({
+      exclude: '.skiplink, .sidebar__collapse-toggle, #wagtail-sidebar, a[href$="dummy-button"]'
+    });
+  });
+});

client/tests/integration/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "wagtail-client-tests-integration",
+  "private": true,
+  "devDependencies": {
+    "@wordpress/jest-puppeteer-axe": "^3.1.0",
+    "expect-puppeteer": "^6.0.0",
+    "jest": "^27.3.1",
+    "jest-junit": "^13.0.0",
+    "puppeteer": "^11.0.0"
+  }
+}

client/tests/integration/setup.js

@@ -0,0 +1,30 @@
+const { mkdir, writeFile } = require('fs').promises;
+const os = require('os');
+const path = require('path');
+const puppeteer = require('puppeteer');
+
+const DIR = path.join(os.tmpdir(), 'jest_puppeteer_global_setup');
+
+/**
+ * Custom Puppeteer setup as documented on https://jestjs.io/docs/puppeteer.
+ */
+module.exports = async () => {
+  const browser = await puppeteer.launch();
+  // store the browser instance so we can teardown it later
+  // this global is only available in the teardown but not in TestEnvironments
+  global.__BROWSER_GLOBAL__ = browser;
+
+  // use the file system to expose the wsEndpoint for TestEnvironments
+  await mkdir(DIR, { recursive: true });
+  await writeFile(path.join(DIR, 'wsEndpoint'), browser.wsEndpoint());
+
+  // Automatically log into the Wagtail admin.
+  const page = await browser.newPage();
+  await page.goto('http://localhost:8000/admin/login/');
+  await page.type('#id_username', 'admin');
+  await page.type('#id_password', 'changeme');
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: 'load' }),
+    page.keyboard.press('Enter'),
+  ]);
+};

client/tests/integration/teardown.js

@@ -0,0 +1,13 @@
+const fs = require('fs').promises;
+const os = require('os');
+const path = require('path');
+
+const DIR = path.join(os.tmpdir(), 'jest_puppeteer_global_setup');
+
+module.exports = async () => {
+  // close the browser instance
+  await global.__BROWSER_GLOBAL__.close();
+
+  // clean-up the wsEndpoint file
+  await fs.rm(DIR, { recursive: true, force: true });
+};

client/tests/integration/users.test.js

@@ -0,0 +1,13 @@
+describe('Users', () => {
+  beforeAll(async () => {
+    await page.goto('http://localhost:8000/admin/users/', { waitUntil: 'load' });
+  });
+
+  it('axe', async () => {
+    const toggle = await page.$('[aria-label="Select all"]');
+    await toggle.click();
+    await expect(page).toPassAxeTests({
+      exclude: '.skiplink, .sidebar__collapse-toggle, #wagtail-sidebar'
+    });
+  });
+});

docs/contributing/developing.rst

@@ -60,7 +60,7 @@ Any Wagtail sites you start up in this virtualenv will now run against this deve
 Testing
 ~~~~~~~
 
-From the root of the Wagtail codebase, run the following command to run all the tests:
+From the root of the Wagtail codebase, run the following command to run all the Python tests:
 
 .. code-block:: console
 
@@ -172,6 +172,26 @@ If your Elasticsearch instance is located somewhere else, you can set the
 
     $ ELASTICSEARCH_URL=http://my-elasticsearch-instance:9200 python runtests.py --elasticsearch
 
+Unit tests for JavaScript
+-------------------------
+
+We use `Jest <https://jestjs.io/>`_ for unit tests of client-side business logic or UI components. From the root of the Wagtail codebase, run the following command to run all the front-end unit tests:
+
+.. code-block:: console
+
+    $ npm run test:unit
+
+Integration tests
+-----------------
+
+Our end-to-end browser testing suite also uses `Jest <https://jestjs.io/>`_, combined with `Puppeteer <https://pptr.dev/>`_. We set this up to be installed separately so as not to increase the installation size of the existing Node tooling. Install the dependencies and run the tests with:
+
+.. code-block:: console
+
+    $ npm --prefix client/tests/integration install
+    $ npm run test:integration
+
+
 Browser and device support
 --------------------------
 
@@ -224,6 +244,7 @@ We want to make Wagtail accessible for users of a wide variety of assistive tech
 We aim for Wagtail to work in those environments. Our development standards ensure that the site is usable with other assistive technologies. In practice, testing with assistive technology can be a daunting task that requires specialised training – here are tools we rely on to help identify accessibility issues, to use during development and code reviews:
 
 * `react-axe <https://github.com/dequelabs/react-axe>`_ integrated directly in our build tools, to identify actionable issues. Logs its results in the browser console.
+* `@wordpress/jest-puppeteer-axe <https://github.com/WordPress/gutenberg/tree/trunk/packages/jest-puppeteer-axe>`_ running Axe checks as part of integration tests.
 * `Axe <https://chrome.google.com/webstore/detail/axe/lhdoppojpmngadmnindnejefpokejbdd>`_ Chrome extension for more comprehensive automated tests of a given page.
 * `Accessibility Insights for Web <https://accessibilityinsights.io/docs/en/web/overview>`_ Chrome extension for semi-automated tests, and manual audits.
 

package.json

@@ -31,7 +31,8 @@
     },
     "testPathIgnorePatterns": [
       "/node_modules/",
-      "/build/"
+      "/build/",
+      "client/tests/integration"
     ],
     "coveragePathIgnorePatterns": [
       "/node_modules/",
@@ -138,6 +139,7 @@
     "test:unit": "jest",
     "test:unit:watch": "jest --watch",
     "test:unit:coverage": "jest --coverage",
+    "test:integration": "./client/tests/integration/node_modules/.bin/jest --config ./client/tests/integration/jest.config.js",
     "storybook": "start-storybook -c client/.storybook -p 6006",
     "build-storybook": "build-storybook -c client/.storybook"
   }

wagtail/tests/manage.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python
+import os
+import sys
+
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "wagtail.tests.settings")
+
+    from django.core.management import execute_from_command_line
+
+    execute_from_command_line(sys.argv)

wagtail/tests/settings.py

@@ -53,6 +53,7 @@ ROOT_URLCONF = 'wagtail.tests.urls'
 STATIC_URL = '/static/'
 
 STATICFILES_FINDERS = (
+    'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
 )
 

wagtail/tests/settings_ui.py

@@ -0,0 +1,9 @@
+from .settings import *  # noqa
+
+
+# Settings meant to run the test suite with Django’s development server, for integration tests.
+DEBUG = True
+
+DATABASES['default']['NAME'] = 'ui_tests.db'  # noqa
+
+WAGTAIL_EXPERIMENTAL_FEATURES = {'slim-sidebar'}

wagtail/tests/urls.py

@@ -1,3 +1,4 @@
+from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.http import HttpResponse
 from django.urls import include, path
 
@@ -40,7 +41,11 @@ urlpatterns = [
     path('testapp/', include(testapp_urls)),
 
     path('fallback/', lambda: HttpResponse('ok'), name='fallback'),
+]
 
+urlpatterns += staticfiles_urlpatterns()
+
+urlpatterns += [
     # For anything not caught by a more specific rule above, hand over to
     # Wagtail's serving mechanism
     path('', include(wagtail_urls)),