diff --git a/.github/workflows/storybook-chromatic.yml b/.github/workflows/storybook-chromatic.yml index 43b15131aff..8d12c36b9e7 100644 --- a/.github/workflows/storybook-chromatic.yml +++ b/.github/workflows/storybook-chromatic.yml @@ -146,15 +146,7 @@ jobs: VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }} STORYBOOK_SKIP_TAGS: 'test-skip,test-skip-${{ matrix.browser }}' run: | - pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT - - - name: Run @playwright/test (legacy, Chromium-only) - if: matrix.browser == 'chromium' && matrix.shard == 1 - env: - # Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot - VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }} - run: | - pnpm test:visual-regression:legacy:ci:$VARIANT + pnpm test:visual-regression:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT - name: Archive failure screenshots if: ${{ failure() }} diff --git a/package.json b/package.json index 24dd8a3ecd5..6509bbff2fa 100644 --- a/package.json +++ b/package.json @@ -21,16 +21,12 @@ "test": "pnpm test:unit && pnpm test:visual-regression", "test:unit": "jest --testPathPattern=frontend/", "jest": "jest", - "test:visual-regression": "docker compose -f docker-compose.playwright.yml run --rm -it --build playwright pnpm test:visual-regression:legacy:docker && pnpm test:visual-regression:stories:docker", - "test:visual-regression:legacy": "docker compose -f docker-compose.playwright.yml run --rm -it --build playwright pnpm test:visual-regression:legacy:docker", - "test:visual-regression:legacy:docker": "STORYBOOK_URL=http://host.docker.internal:6006 playwright test -u", - "test:visual-regression:legacy:ci:update": "playwright test -u", - "test:visual-regression:legacy:ci:verify": "playwright test", - "test:visual-regression:stories": "rm -rf frontend/__snapshots__/__failures__/ && docker compose -f docker-compose.playwright.yml run --rm -it --build playwright pnpm test:visual-regression:stories:docker", - "test:visual-regression:stories:docker": "NODE_OPTIONS=--max-old-space-size=6144 test-storybook -u --no-index-json --browsers chromium webkit --url http://host.docker.internal:6006", - "test:visual-regression:stories:local": "NODE_OPTIONS=--max-old-space-size=6144 test-storybook -u --no-index-json --browsers chromium webkit --url http://localhost:6006", - "test:visual-regression:stories:ci:update": "test-storybook -u --no-index-json --maxWorkers=2", - "test:visual-regression:stories:ci:verify": "test-storybook --ci --no-index-json --maxWorkers=2", + "test:visual-regression": "pnpm test:visual-regression:docker", + "test:visual-regression": "rm -rf frontend/__snapshots__/__failures__/ && docker compose -f docker-compose.playwright.yml run --rm -it --build playwright pnpm test:visual-regression:docker", + "test:visual-regression:docker": "NODE_OPTIONS=--max-old-space-size=6144 test-storybook -u --no-index-json --browsers chromium webkit --url http://host.docker.internal:6006", + "test:visual-regression:local": "NODE_OPTIONS=--max-old-space-size=6144 test-storybook -u --no-index-json --browsers chromium webkit --url http://localhost:6006", + "test:visual-regression:ci:update": "test-storybook -u --no-index-json --maxWorkers=2", + "test:visual-regression:ci:verify": "test-storybook --ci --no-index-json --maxWorkers=2", "start": "concurrently -n ESBUILD,TYPEGEN -c yellow,green \"pnpm start-http\" \"pnpm run typegen:watch\"", "start-http": "pnpm clean && pnpm copy-scripts && node frontend/build.mjs --dev", "start-docker": "pnpm start-http --host 0.0.0.0", diff --git a/playwright/e2e-vrt/README.md b/playwright/e2e-vrt/README.md deleted file mode 100644 index 56a1ae14664..00000000000 --- a/playwright/e2e-vrt/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Visual Regression Tests - -We're using Playwright to run visual regression tests against Storybook. To create reference images that are consistent between development and CI we run Playwright in an Ubuntu Docker container. - -## Workflow - -### Writing or updating a test - -0. Use fixtures - - _tbd_ - -1. Perform the actions to get to the state - - - Get inspiration from one of the existing tests. Lemon Button and Insights scene are good examples. - - - Use Playwright's [codegen feature](https://playwright.dev/docs/codegen-intro) to record user interactions interactively e.g. `pnpm playwright codegen "http://localhost:6006/iframe.html?args=&id=scenes-app-insights--trends-line&viewMode=story"`. - - - Use Playwright's [debug mode](https://playwright.dev/docs/debug) to inspect an existing test in a headed browser e.g. `pnpm playwright test --debug e2e-vrt/scenes-app/mytest.spec.ts:17:5`. Sprinkle in `await page.pause()` in your test to stop at specific lines. - -2. Add screenshot expectations: - - - Capture the whole page (we rarely use this): - - ```ts - await expect(page).toHaveScreenshot({ fullPage: true }) - ``` - - - Capture content within a container element: - - ```ts - const locator = page.locator('#storybook-root') - await expect(locator).toHaveScreenshot() - ``` - - - Suggested Storybook container elements: - - `#root` for components and - - `main` for scenes - -3. Generate the reference images (you need to have Storybook running locally, i.e. on the Docker host machine): - - ```sh - docker compose -f docker-compose.playwright.yml run -it --build -e STORYBOOK_URL=http://host.docker.internal:6006 playwright pnpm test:visual-regression - ``` - - > If your `docker compose version` is lower than 2.13, you won't be able to use the `--build` flag. Instead, first run `docker compose -f docker-compose.playwright.yml build playwright`, and then the above `run` command - without the `--build` arg. - - Open the generated report locally with `pnpm dlx playwright show-report` to see test results (they are mounted local in docker volume) - -### Renaming or deleting tests - -When deleting or renaming a test, (re-)move the respective reference images as well or delete `rm -rf playwright/**/*.png` and re-create (see above) them. - -## Troubleshooting - -### The CI run on GitHub fails for any reason - -Troubleshoot by viewing the Playwright report: Click on "Details" next to the failing workflow, click on "Summary" and download the artifact file. Extract this file and navigate to it in a terminal to then run `pnpm dlx playwright show-report`. - -### Your locally generated images, don't verify CI checks - -GitHub is running tests against a temporary merge commit (to ensure tests still pass after being merged), meaning any changes currently in master will be present in the images generated in the CI run. If you suspect this is the case, merge master into your branch and push again. - -### The screenshots look wrong and I want to debug it live - -If you run any playwright command with `PWDEBUG=1` then it will run in "headfull" mode so that you can see the browser and look into what is going on diff --git a/playwright/e2e-vrt/lemon-ui/LemonButton.spec.ts b/playwright/e2e-vrt/lemon-ui/LemonButton.spec.ts deleted file mode 100644 index 1550aa9a9bd..00000000000 --- a/playwright/e2e-vrt/lemon-ui/LemonButton.spec.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { toId } from '../../helpers/storybook' -import { test } from '../../fixtures/storybook' - -test.describe('Lemon Button', () => { - // TODO: Remove when our Storybook test runner supports play tests - test('displays hover state correctly', async ({ storyPage }) => { - await storyPage.goto(toId('Lemon UI/Lemon Button', 'Default')) - await storyPage.expectComponentScreenshot({ pseudo: { hover: true } }) - }) - - test('displays disabled reason correctly', async ({ storyPage }) => { - await storyPage.goto(toId('Lemon UI/Lemon Button', 'Disabled With Reason')) - await storyPage.storyRoot.locator('.LemonButton').nth(2).hover() - await storyPage.page.getByRole('tooltip').waitFor() - await storyPage.expectComponentScreenshot({ pseudo: { hover: true } }) - }) -}) diff --git a/playwright/e2e-vrt/lemon-ui/LemonButton.spec.ts-snapshots/Lemon-Button-displays-hover-state-correctly-1-chromium-linux.png b/playwright/e2e-vrt/lemon-ui/LemonButton.spec.ts-snapshots/Lemon-Button-displays-hover-state-correctly-1-chromium-linux.png deleted file mode 100644 index 45791568478..00000000000 Binary files a/playwright/e2e-vrt/lemon-ui/LemonButton.spec.ts-snapshots/Lemon-Button-displays-hover-state-correctly-1-chromium-linux.png and /dev/null differ diff --git a/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts b/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts deleted file mode 100644 index 59f4ab378c7..00000000000 --- a/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { toId } from '../../helpers/storybook' -import { test } from '../../fixtures/storybook' - -test.describe('Persons Modal', () => { - test('displays list correctly', async ({ storyPage }) => { - await storyPage.goto(toId('Scenes-App/Persons Modal', 'Persons Modal')) - await storyPage.expectComponentScreenshot() - // Expand first person to see properties - await storyPage.page.click('[data-attr=persons-modal-expand-018339dc-735e-0000-fd6a-963eda28b90d]') - await storyPage.expectComponentScreenshot() - }) -}) diff --git a/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts-snapshots/Persons-Modal-displays-list-correctly-1-chromium-linux.png b/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts-snapshots/Persons-Modal-displays-list-correctly-1-chromium-linux.png deleted file mode 100644 index 380f1eadfd0..00000000000 Binary files a/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts-snapshots/Persons-Modal-displays-list-correctly-1-chromium-linux.png and /dev/null differ diff --git a/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts-snapshots/Persons-Modal-displays-list-correctly-2-chromium-linux.png b/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts-snapshots/Persons-Modal-displays-list-correctly-2-chromium-linux.png deleted file mode 100644 index 95b8f44be91..00000000000 Binary files a/playwright/e2e-vrt/scenes-app/PersonsModal.spec.ts-snapshots/Persons-Modal-displays-list-correctly-2-chromium-linux.png and /dev/null differ diff --git a/playwright/e2e-vrt/scenes-app/insights.spec.ts b/playwright/e2e-vrt/scenes-app/insights.spec.ts deleted file mode 100644 index 38d0b736b85..00000000000 --- a/playwright/e2e-vrt/scenes-app/insights.spec.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { toId } from '../../helpers/storybook' -import { test, expect } from '../../fixtures/storybook' - -test.describe('tooltip', () => { - // skipped because this flaps like a fish out of water - test('displays correctly', async ({ page, storyPage }) => { - await storyPage.goto(toId('Scenes-App/Insights', 'Trends Line')) - - // hover over the graph to show the tooltip - await page.locator('canvas').hover({ - position: { - x: 412, - y: 150, - }, - }) - - const tooltip = await page.locator('.InsightTooltip') - await expect(tooltip).toHaveScreenshot({ maxDiffPixelRatio: 0.03 }) - }) -}) - -test.describe('annotations popover', () => { - // skipped because this flaps like a fish out of water - test('displays correctly', async ({ page, storyPage }) => { - await storyPage.goto(toId('Scenes-App/Insights', 'Trends Line')) - - // hover over the graph to show the annotations overlay - await page.locator('.AnnotationsOverlay > button:nth-child(4)').hover() - - const popover = await page.locator('.AnnotationsPopover') - await expect(popover).toHaveScreenshot({ maxDiffPixelRatio: 0.03 }) - }) -}) diff --git a/playwright/e2e-vrt/scenes-app/insights.spec.ts-snapshots/annotations-popover-displays-correctly-1-chromium-linux.png b/playwright/e2e-vrt/scenes-app/insights.spec.ts-snapshots/annotations-popover-displays-correctly-1-chromium-linux.png deleted file mode 100644 index 76e255667d9..00000000000 Binary files a/playwright/e2e-vrt/scenes-app/insights.spec.ts-snapshots/annotations-popover-displays-correctly-1-chromium-linux.png and /dev/null differ diff --git a/playwright/e2e-vrt/scenes-app/insights.spec.ts-snapshots/tooltip-displays-correctly-1-chromium-linux.png b/playwright/e2e-vrt/scenes-app/insights.spec.ts-snapshots/tooltip-displays-correctly-1-chromium-linux.png deleted file mode 100644 index 0fef7bf600e..00000000000 Binary files a/playwright/e2e-vrt/scenes-app/insights.spec.ts-snapshots/tooltip-displays-correctly-1-chromium-linux.png and /dev/null differ diff --git a/playwright/fixtures/storybook.ts b/playwright/fixtures/storybook.ts deleted file mode 100644 index e93d743bc05..00000000000 --- a/playwright/fixtures/storybook.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { test as base } from '@playwright/test' - -import { StorybookStoryPage } from '../pages/storybook' - -type StorybookFixtures = { - storyPage: StorybookStoryPage -} - -export const test = base.extend({ - storyPage: async ({ page }, use) => { - const storyPage = new StorybookStoryPage(page) - await use(storyPage) - }, -}) - -export { expect } from '@playwright/test' diff --git a/playwright/helpers/storybook.ts b/playwright/helpers/storybook.ts deleted file mode 100644 index 6b18c22b081..00000000000 --- a/playwright/helpers/storybook.ts +++ /dev/null @@ -1,12 +0,0 @@ -// Remove punctuation and illegal characters -const sanitize = (string: string): string => { - return string - .toLowerCase() - .replace(/[ ’–—―′¿'`~!@#$%^&*()_|+\-=?;:'",.<>\{\}\[\]\\\/]/gi, '-') - .replace(/-+/g, '-') - .replace(/^-+/, '') - .replace(/-+$/, '') -} - -// Generate a story id from a story kind and name -export const toId = (kind: string, name?: string): string => `${sanitize(kind)}${name ? `--${sanitize(name)}` : ''}` diff --git a/playwright/pages/storybook.ts b/playwright/pages/storybook.ts deleted file mode 100644 index 4f990c81a3c..00000000000 --- a/playwright/pages/storybook.ts +++ /dev/null @@ -1,72 +0,0 @@ -import { expect, Locator, Page } from '@playwright/test' - -const STORYBOOK_URL: string = process.env.STORYBOOK_URL || 'http://localhost:6006' - -const PSEUDO_STATES = { - hover: 'hover', - active: 'active', - focusVisible: 'focus-visible', - focusWithin: 'focus-within', - focus: 'focus', - visited: 'visited', - link: 'link', - target: 'target', -} - -type ComponentScreenshotConfig = { - pseudo?: Partial> -} - -export class StorybookStoryPage { - readonly page: Page - readonly storyRoot: Locator - - constructor(page: Page) { - this.page = page - this.storyRoot = page.locator('#storybook-root') - } - - async goto(storyId: string): Promise { - const storyUrl = `${STORYBOOK_URL}/iframe.html?id=${storyId}&viewMode=story` - await this.page.goto(storyUrl, { waitUntil: 'networkidle' }) - } - - async resizeToMobile(): Promise { - await this.page.setViewportSize({ width: 375, height: 667 }) // iPhone 6/7/8 - } - - async expectFullPageScreenshot(): Promise { - await expect(this.page).toHaveScreenshot({ maxDiffPixelRatio: 0.01 }) - } - - async expectComponentScreenshot({ pseudo } = {} as ComponentScreenshotConfig): Promise { - const pseudoClasses = Object.entries(pseudo || {}).flatMap(([state, enabled]) => { - return enabled ? `pseudo-${PSEUDO_STATES[state]}` : [] - }) - - await this.page.evaluate( - ([pseudoClasses]) => { - const rootEl = document.getElementById('storybook-root') - - if (rootEl) { - // don't expand the container element to limit the screenshot - // to the component's size - rootEl.style.display = 'inline-block' - - // add classes for pseudo states generated by - // storybook-addon-pseudo-states - pseudoClasses.forEach((c) => { - rootEl.classList.add(c) - }) - } - - // make the body transparent to take the screenshot - // without background - document.body.style.background = 'transparent' - }, - [pseudoClasses] - ) - - await expect(this.storyRoot).toHaveScreenshot({ omitBackground: true, maxDiffPixelRatio: 0.01 }) - } -}