test(frontend): Automatically test Storybook stories with snapshots (#13839)

.dockerignore

@@ -15,6 +15,7 @@
 !postcss.config.js
 !playwright.config.ts
 !.kearc
+!.storybook
 !tsconfig.json
 !frontend/@posthog
 !frontend/src

.github/workflows/ci-e2e-vrt.yml

@@ -1,52 +0,0 @@
-name: E2E Visual Regression Tests
-
-on:
-    pull_request:
-    push:
-        branches:
-            - master
-
-jobs:
-    playwright:
-        name: Visual Regression Tests
-        runs-on: ubuntu-20.04
-        container:
-            image: mcr.microsoft.com/playwright:v1.28.1-focal
-        steps:
-            - uses: actions/checkout@v3
-
-            - name: Install pnpm
-              uses: pnpm/action-setup@v2
-              with:
-                  version: 7.x.x
-
-            - name: Set up Node.js
-              uses: actions/setup-node@v3
-              with:
-                  node-version: 18
-                  cache: pnpm
-
-            - name: Install package.json dependencies with pnpm
-              run: pnpm install --frozen-lockfile
-
-            - name: Install CI utilities with pnpm
-              run: pnpm install concurrently http-server wait-on
-
-            - name: Build Storybook
-              run: pnpm build-storybook --quiet
-
-            - name: Serve Storybook and run tests
-              run: |
-                  pnpm exec concurrently -k -s first -n "SRVR,TEST" -c "magenta,blue" \
-                    "pnpm exec http-server storybook-static --port 6006" \
-                    "pnpm exec wait-on http://127.0.0.1:6006 --timeout 60 && pnpm exec playwright test"
-
-            - name: Upload Playwright report
-              uses: actions/upload-artifact@v3
-              if: always()
-              with:
-                  name: playwright-report
-                  path: |
-                      playwright-report/
-                      test-results/
-                  retention-days: 7

.github/workflows/ci-frontend.yml

@@ -66,10 +66,10 @@ jobs:
             - run: pnpm install --frozen-lockfile
             - id: set-test-chunks
               name: Set Chunks
-              # Looks at the output of 'pnpm test -- --listTests --json'
+              # Looks at the output of 'pnpm test:unit -- --listTests --json'
               # Take the 5th line of the output (the first two are pnpm non-sense)
               # Split the test into 3 parts. To increase the number split change the denominator in `length / 3`
-              run: echo "test-chunks=$(pnpm test -- --listTests --json | sed -n 5p | jq -cM '[_nwise(length / 3 | ceil)]')" >> $GITHUB_OUTPUT
+              run: echo "test-chunks=$(pnpm test:unit --listTests --json | sed -n 5p | jq -cM '[_nwise(length / 3 | ceil)]')" >> $GITHUB_OUTPUT
             - id: set-test-chunk-ids
               name: Set Chunk IDs
               run: echo "test-chunk-ids=$(echo $CHUNKS | jq -cM 'to_entries | map(.key)')" >> $GITHUB_OUTPUT
@@ -106,7 +106,57 @@ jobs:
 
             - name: Test with Jest
               # set maxWorkers or Jest only uses 1 CPU in GitHub Actions
-              run: echo $CHUNKS | jq '.[${{ matrix.chunk }}] | .[] | @text' | xargs pnpm test -- --maxWorkers=2
+              run: pnpm test:unit --maxWorkers=2 $(echo $CHUNKS | jq '.[${{ matrix.chunk }}] | .[] | @text')
               env:
                   NODE_OPTIONS: --max-old-space-size=6144
                   CHUNKS: ${{ needs.jest-setup.outputs['test-chunks'] }}
+
+    visual-regression:
+        name: Visual regression tests
+        runs-on: ubuntu-20.04
+        container:
+            image: mcr.microsoft.com/playwright:v1.29.2-focal
+        env:
+            CYPRESS_INSTALL_BINARY: '0'
+            NODE_OPTIONS: --max_old_space_size=4096
+        steps:
+            - uses: actions/checkout@v3
+
+            - name: Install pnpm
+              uses: pnpm/action-setup@v2
+              with:
+                  version: 7.x.x
+
+            - name: Set up Node.js
+              uses: actions/setup-node@v3
+              with:
+                  node-version: 18
+                  cache: pnpm
+
+            - name: Install package.json dependencies with pnpm
+              run: pnpm install --frozen-lockfile
+
+            - name: Install CI utilities with pnpm
+              run: pnpm install http-server wait-on
+
+            - name: Build Storybook
+              run: pnpm build-storybook --quiet # Silence since progress logging results in a massive wall of spam
+
+            - name: Serve Storybook in the background
+              run: pnpm exec http-server storybook-static --port 6006 --silent &
+
+            - name: Compare current Storybook to the baseline
+              run: |
+                  pnpm wait-on http://127.0.0.1:6006 --timeout 60
+                  pnpm test:visual-regression:ci
+
+            - name: Upload Playwright report and diffs
+              uses: actions/upload-artifact@v3
+              if: always() # We want the report even if there's no failure
+              with:
+                  name: playwright-report
+                  path: |
+                      playwright-report/
+                      test-results/
+                      frontend/__snapshots__/__diff_output__/
+                  retention-days: 7

.gitignore

@@ -17,6 +17,7 @@ frontend/.cache/
 .mypy_cache
 frontend/dist/
 frontend/types/
+frontend/__snapshots__/__diff_output__/
 *Type.ts
 frontend/pnpm-error.log
 frontend/tmp

.husky/pre-commit

@@ -1,4 +1,13 @@
 #!/bin/sh
 . "$(dirname "$0")/_/husky.sh"
 
+# Check if staged files contain any added or modified PNGs
+if git diff --cached --name-status | grep '^[AM]' | grep -q '.png$'; then
+  # Error if OptiPNG is not installed
+    if ! command -v optipng >/dev/null; then
+        echo "PNG files must be optimized before being committed, but OptiPNG is not installed! Fix this with \`brew/apt install optipng\`."
+        exit 1
+    fi
+fi
+
 pnpm lint-staged

.storybook/decorators/withSnapshotsDisabled.tsx

@@ -0,0 +1,10 @@
+import { DecoratorFn } from '@storybook/react'
+
+/** Workaround for https://github.com/storybookjs/test-runner/issues/74 */
+// TODO: Smoke-test all the stories by removing this decorator, once all the stories pass
+export const withSnapshotsDisabled: DecoratorFn = (Story, { parameters }) => {
+    if (parameters?.chromatic?.disableSnapshot && navigator.userAgent.includes('StorybookTestRunner')) {
+        return <>Disabled for Test Runner</>
+    }
+    return <Story />
+}

.storybook/preview.tsx

@@ -7,6 +7,7 @@ import { getStorybookAppContext } from './app-context'
 import { withKea } from './decorators/withKea'
 import { withMockDate } from './decorators/withMockDate'
 import { defaultMocks } from '~/mocks/handlers'
+import { withSnapshotsDisabled } from './decorators/withSnapshotsDisabled'
 
 const setupMsw = () => {
     // Make sure the msw worker is started
@@ -27,9 +28,9 @@ const setupPosthogJs = () => {
 }
 setupPosthogJs()
 
-// Setup storybook global parameters. See https://storybook.js.org/docs/react/writing-stories/parameters#global-parameters
+/** Storybook global parameters. See https://storybook.js.org/docs/react/writing-stories/parameters#global-parameters */
 export const parameters = {
-    chromatic: { disableSnapshot: true },
+    chromatic: { disableSnapshot: true }, // TODO: Make snapshots the default, instead disable them on a per-story basis
     actions: { argTypesRegex: '^on[A-Z].*', disabled: true },
     controls: {
         matchers: {
@@ -66,6 +67,7 @@ export const parameters = {
 
 // Setup storybook global decorators. See https://storybook.js.org/docs/react/writing-stories/decorators#global-decorators
 export const decorators: Meta['decorators'] = [
+    withSnapshotsDisabled,
     // Make sure the msw service worker is started, and reset the handlers to defaults.
     withKea,
     // Allow us to time travel to ensure our stories don't change over time.

.storybook/test-runner.ts

@@ -0,0 +1,73 @@
+import { toMatchImageSnapshot } from 'jest-image-snapshot'
+import { getStoryContext, TestRunnerConfig, TestContext } from '@storybook/test-runner'
+import { Locator, Page, LocatorScreenshotOptions } from 'playwright-core'
+
+const customSnapshotsDir = `${process.cwd()}/frontend/__snapshots__`
+
+async function expectStoryToMatchFullPageSnapshot(page: Page, context: TestContext): Promise<void> {
+    await expectLocatorToMatchStorySnapshot(page, context)
+}
+
+async function expectStoryToMatchSceneSnapshot(page: Page, context: TestContext): Promise<void> {
+    await expectLocatorToMatchStorySnapshot(page.locator('.main-app-content'), context)
+}
+
+async function expectStoryToMatchComponentSnapshot(page: Page, context: TestContext): Promise<void> {
+    await page.evaluate(() => {
+        const rootEl = document.getElementById('root')
+
+        if (rootEl) {
+            // don't expand the container element to limit the screenshot
+            // to the component's size
+            rootEl.style.display = 'inline-block'
+        }
+
+        // make the body transparent to take the screenshot
+        // without background
+        document.body.style.background = 'transparent'
+    })
+
+    await expectLocatorToMatchStorySnapshot(page.locator('#root'), context, { omitBackground: true })
+}
+
+async function expectLocatorToMatchStorySnapshot(
+    locator: Locator | Page,
+    context: TestContext,
+    options?: LocatorScreenshotOptions
+): Promise<void> {
+    const image = await locator.screenshot(options)
+    expect(image).toMatchImageSnapshot({
+        customSnapshotsDir,
+        customSnapshotIdentifier: context.id,
+    })
+}
+
+module.exports = {
+    setup() {
+        expect.extend({ toMatchImageSnapshot })
+    },
+    async postRender(page, context) {
+        const storyContext = await getStoryContext(page, context)
+
+        await page.evaluate(() => {
+            // Stop all animations for consistent snapshots
+            document.body.classList.add('dangerously-stop-all-animations')
+        })
+
+        // Wait for the network to be idle for up to 500 ms, to allow assets like images to load. This is suboptimal,
+        // because `networkidle` is not resolved reliably here, so we might wait for the full timeout - but it works.
+        await Promise.race([page.waitForLoadState('networkidle'), page.waitForTimeout(500)])
+
+        if (!storyContext.parameters?.chromatic?.disableSnapshot) {
+            if (storyContext.parameters?.layout === 'fullscreen') {
+                if (storyContext.parameters.testRunner?.includeNavigation) {
+                    await expectStoryToMatchFullPageSnapshot(page, context)
+                } else {
+                    await expectStoryToMatchSceneSnapshot(page, context)
+                }
+            } else {
+                await expectStoryToMatchComponentSnapshot(page, context)
+            }
+        }
+    },
+} as TestRunnerConfig

Dockerfile.playwright

@@ -3,12 +3,20 @@
 # We do this to ensure our reference images for visual regression tests are the same during development and in CI.
 #
 
-FROM mcr.microsoft.com/playwright:v1.28.1-focal
+FROM mcr.microsoft.com/playwright:v1.29.2-focal
 
 WORKDIR /work
 
 RUN npm install -g pnpm
 
-RUN pnpm install @playwright/test@1.28.1
+COPY package.json pnpm-lock.yaml ./
 
-COPY playwright.config.ts ./
+ENV CYPRESS_INSTALL_BINARY=0
+
+RUN pnpm install
+
+COPY playwright.config.ts webpack.config.js babel.config.js tsconfig.json ./
+
+COPY .storybook/ .storybook/
+
+COPY frontend/ frontend/

docker-compose.playwright.yml

@@ -5,7 +5,7 @@ services:
             dockerfile: Dockerfile.playwright
         network_mode: host
         volumes:
+            - './frontend/__snapshots__:/work/frontend/__snapshots__'
             - './playwright:/work/playwright'
             - './playwright-report:/work/playwright-report'
             - './test-results:/work/test-results'
-        # profiles: [playwright]