mirror of
https://github.com/PostHog/posthog.git
synced 2024-11-25 11:17:50 +01:00
3908adb4b2
* chore: make visual regression less sensitive to differences * Update UI snapshots for `chromium` (1) * git diff only after optipng has run * git diff only after optipng has run * Update UI snapshots for `chromium` (1) * Update .storybook/test-runner.ts * text * no need to run lint staged... we've already run optipng --------- Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
284 lines
15 KiB
YAML
284 lines
15 KiB
YAML
name: Storybook
|
|
|
|
on:
|
|
pull_request:
|
|
paths: # Only run if the frontend has changed
|
|
- 'frontend/**'
|
|
- '.storybook/**'
|
|
- 'package.json'
|
|
- '.github/workflows/storybook-chromatic.yml'
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
# This is so that the workflow run isn't canceled when a snapshot update is pushed within it by posthog-bot
|
|
# We do however cancel from container-images-ci.yml if a commit is pushed by someone OTHER than posthog-bot
|
|
cancel-in-progress: false
|
|
|
|
jobs:
|
|
storybook-chromatic:
|
|
name: Publish to Chromatic
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
if: github.event.pull_request.head.repo.full_name == github.repository # Don't run on forks
|
|
outputs:
|
|
storybook-url: ${{ steps.publish.outputs.storybookUrl }}
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0 # 👈 Required to retrieve git history (https://www.chromatic.com/docs/github-actions)
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v2
|
|
with:
|
|
version: 8.x.x
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@v3
|
|
with:
|
|
node-version: 18
|
|
cache: pnpm
|
|
|
|
- name: Install dependencies and Chromatic
|
|
run: pnpm i -D chromatic
|
|
|
|
- name: Publish to Chromatic
|
|
uses: chromaui/action@v1
|
|
id: publish
|
|
with:
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
# 👇 Chromatic projectToken, refer to the manage page to obtain it.
|
|
projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
|
|
|
|
visual-regression:
|
|
name: Visual regression tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 30
|
|
container:
|
|
image: mcr.microsoft.com/playwright:v1.29.2 # Same as @storybook/test-runner@0.13's dependency
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
browser: ['chromium', 'webkit']
|
|
shard: [1, 2]
|
|
env:
|
|
SHARD_COUNT: '2'
|
|
CYPRESS_INSTALL_BINARY: '0'
|
|
NODE_OPTIONS: --max-old-space-size=6144
|
|
JEST_IMAGE_SNAPSHOT_TRACK_OBSOLETE: '1' # Remove obsolete snapshots
|
|
OPT_OUT_CAPTURE: 1
|
|
outputs:
|
|
# The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs
|
|
chromium-1-added: ${{ steps.diff.outputs.chromium-1-added }}
|
|
chromium-1-modified: ${{ steps.diff.outputs.chromium-1-modified }}
|
|
chromium-1-deleted: ${{ steps.diff.outputs.chromium-1-deleted }}
|
|
chromium-1-total: ${{ steps.diff.outputs.chromium-1-total }}
|
|
chromium-1-commitHash: ${{ steps.commit-hash.outputs.chromium-1-commitHash }}
|
|
chromium-2-added: ${{ steps.diff.outputs.chromium-2-added }}
|
|
chromium-2-modified: ${{ steps.diff.outputs.chromium-2-modified }}
|
|
chromium-2-deleted: ${{ steps.diff.outputs.chromium-2-deleted }}
|
|
chromium-2-total: ${{ steps.diff.outputs.chromium-2-total }}
|
|
chromium-2-commitHash: ${{ steps.commit-hash.outputs.chromium-2-commitHash }}
|
|
webkit-1-added: ${{ steps.diff.outputs.webkit-1-added }}
|
|
webkit-1-modified: ${{ steps.diff.outputs.webkit-1-modified }}
|
|
webkit-1-deleted: ${{ steps.diff.outputs.webkit-1-deleted }}
|
|
webkit-1-total: ${{ steps.diff.outputs.webkit-1-total }}
|
|
webkit-1-commitHash: ${{ steps.commit-hash.outputs.webkit-1-commitHash }}
|
|
webkit-2-added: ${{ steps.diff.outputs.webkit-2-added }}
|
|
webkit-2-modified: ${{ steps.diff.outputs.webkit-2-modified }}
|
|
webkit-2-deleted: ${{ steps.diff.outputs.webkit-2-deleted }}
|
|
webkit-2-total: ${{ steps.diff.outputs.webkit-2-total }}
|
|
webkit-2-commitHash: ${{ steps.commit-hash.outputs.webkit-2-commitHash }}
|
|
firefox-1-added: ${{ steps.diff.outputs.firefox-1-added }}
|
|
firefox-1-modified: ${{ steps.diff.outputs.firefox-1-modified }}
|
|
firefox-1-deleted: ${{ steps.diff.outputs.firefox-1-deleted }}
|
|
firefox-1-total: ${{ steps.diff.outputs.firefox-1-total }}
|
|
firefox-1-commitHash: ${{ steps.commit-hash.outputs.firefox-1-commitHash }}
|
|
firefox-2-added: ${{ steps.diff.outputs.firefox-2-added }}
|
|
firefox-2-modified: ${{ steps.diff.outputs.firefox-2-modified }}
|
|
firefox-2-deleted: ${{ steps.diff.outputs.firefox-2-deleted }}
|
|
firefox-2-total: ${{ steps.diff.outputs.firefox-2-total }}
|
|
firefox-2-commitHash: ${{ steps.commit-hash.outputs.firefox-2-commitHash }}
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 1
|
|
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
|
ref: ${{ github.event.pull_request.head.ref }}
|
|
# Use PostHog Bot token when not on forks to enable proper snapshot updating
|
|
token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
|
|
|
|
- name: Install pnpm
|
|
uses: pnpm/action-setup@v2
|
|
with:
|
|
version: 8.x.x
|
|
|
|
- name: Set up Node.js
|
|
uses: buildjet/setup-node@v3
|
|
with:
|
|
node-version: 18
|
|
cache: pnpm
|
|
|
|
- name: Install package.json dependencies with pnpm
|
|
run: pnpm install --frozen-lockfile
|
|
|
|
- name: Install CI utilities with pnpm
|
|
run: pnpm install http-server wait-on
|
|
|
|
- name: Build Storybook
|
|
run: pnpm build-storybook --quiet # Silence since progress logging results in a massive wall of spam
|
|
|
|
- name: Serve Storybook in the background
|
|
run: |
|
|
pnpm exec http-server storybook-static --port 6006 --silent &
|
|
pnpm wait-on http://127.0.0.1:6006 --timeout 60 # Wait for the server to be ready
|
|
|
|
- name: Run @storybook/test-runner
|
|
env:
|
|
# Solving this bug by overriding $HOME: https://github.com/microsoft/playwright/issues/6500
|
|
HOME: /root
|
|
# Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
|
|
VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
|
|
run: |
|
|
pnpm test:visual-regression:stories:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT
|
|
|
|
- name: Run @playwright/test (legacy, Chromium-only)
|
|
if: matrix.browser == 'chromium' && matrix.shard == 1
|
|
env:
|
|
# Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot
|
|
VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }}
|
|
run: |
|
|
pnpm test:visual-regression:legacy:ci:$VARIANT
|
|
|
|
- name: Count and optimize updated snapshots
|
|
id: diff
|
|
# Skip on forks
|
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
|
run: |
|
|
git config --global --add safe.directory '*' # Calm git down about file ownership
|
|
git diff --name-status frontend/__snapshots__/ # For debugging
|
|
ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
|
|
MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
|
|
DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
|
|
TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)
|
|
|
|
# If added or modified, run OptiPNG
|
|
if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
|
|
echo "Snapshots updated ($ADDED new, $MODIFIED changed), running OptiPNG"
|
|
apt update && apt install -y optipng
|
|
optipng -clobber -o4 -strip all
|
|
|
|
# we don't want to _always_ run OptiPNG
|
|
# so, we run it after checking for a diff
|
|
# but, the files we diffed might then be changed by OptiPNG
|
|
# and as a result they might no longer be different...
|
|
|
|
# we check again
|
|
git diff --name-status frontend/__snapshots__/ # For debugging
|
|
ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l)
|
|
MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l)
|
|
DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l)
|
|
TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l)
|
|
|
|
if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then
|
|
echo "Snapshots updated ($ADDED new, $MODIFIED changed), _even after_ running OptiPNG"
|
|
git add frontend/__snapshots__/ playwright/
|
|
fi
|
|
fi
|
|
|
|
echo "${{ matrix.browser }}-${{ matrix.shard }}-added=$ADDED" >> $GITHUB_OUTPUT
|
|
echo "${{ matrix.browser }}-${{ matrix.shard }}-modified=$MODIFIED" >> $GITHUB_OUTPUT
|
|
echo "${{ matrix.browser }}-${{ matrix.shard }}-deleted=$DELETED" >> $GITHUB_OUTPUT
|
|
echo "${{ matrix.browser }}-${{ matrix.shard }}-total=$TOTAL" >> $GITHUB_OUTPUT
|
|
|
|
- name: Commit updated snapshots
|
|
uses: EndBug/add-and-commit@v9
|
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
|
id: commit
|
|
with:
|
|
add: '["frontend/__snapshots__/", "playwright/"]'
|
|
message: 'Update UI snapshots for `${{ matrix.browser }}` (${{ matrix.shard }})'
|
|
pull: --rebase --autostash # Make sure we're up to date with other browsers' updates
|
|
default_author: github_actions
|
|
github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
|
|
|
|
- name: Add commit hash to outputs, including browser name
|
|
id: commit-hash
|
|
if: steps.commit.outputs.pushed == 'true'
|
|
run: echo "${{ matrix.browser }}-${{ matrix.shard }}-commitHash=${{ steps.commit.outputs.commit_long_sha }}" >> $GITHUB_OUTPUT
|
|
|
|
visual-regression-summary:
|
|
name: Summarize visual regression tests
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
needs: visual-regression
|
|
if: always() # Run even if visual-regression fails for one (or more) of the browsers
|
|
steps:
|
|
- name: Post comment about updated snapshots
|
|
if: github.event.pull_request.head.repo.full_name == github.repository
|
|
uses: actions/github-script@v6
|
|
with:
|
|
github-token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
|
|
script: |
|
|
const BROWSERS = ['chromium', 'webkit']
|
|
|
|
const diffJobOutputs = ${{ toJson(needs.visual-regression.outputs) }}
|
|
const summaryDiff = { total: 0, added: 0, modified: 0, deleted: 0 }
|
|
const diffByBrowser = Object.fromEntries(BROWSERS.map(browser => [browser, {
|
|
total: 0, added: 0, modified: 0, deleted: 0, commitHashes: []
|
|
}]))
|
|
for (const [key, rawValue] of Object.entries(diffJobOutputs)) {
|
|
// Split e.g. 'chromium-1-commitHash' into ['chromium', '1' 'commitHash']
|
|
const [browser, shardNumber, diffKey] = key.split('-')
|
|
// Sum up the counts - but not the commit hash
|
|
if (diffKey === 'commitHash') {
|
|
diffByBrowser[browser].commitHashes.push([parseInt(shardNumber), rawValue])
|
|
} else {
|
|
const value = parseInt(rawValue)
|
|
diffByBrowser[browser][diffKey] += value
|
|
summaryDiff[diffKey] += value
|
|
}
|
|
}
|
|
|
|
for (const browser of BROWSERS) {
|
|
if (diffByBrowser[browser]?.total === undefined) {
|
|
diffByBrowser[browser] = null // Null means failure
|
|
}
|
|
}
|
|
|
|
if (summaryDiff.total === 0) {
|
|
console.log('No changes were made, skipping comment')
|
|
return
|
|
}
|
|
|
|
const diffByBrowserDisplay = Object.entries(diffByBrowser).map(([browser, diff]) => {
|
|
if (!diff) {
|
|
return `- \`${browser}\`: failed`
|
|
}
|
|
const { added: a, modified: m, deleted: d, commitHashes } = diff
|
|
const b = a + m + d > 0 ? '**' : '' // Bold list item if there were changes
|
|
let extraInfo = ''
|
|
if (b) {
|
|
const commitInfo = commitHashes.map(
|
|
([shardNumber, commitHash]) =>
|
|
`[diff for shard ${shardNumber}](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${commitHash})`
|
|
).join(', ') || "wasn't pushed!"
|
|
extraInfo = ` (${commitInfo})`
|
|
}
|
|
return `- ${b}\`${browser}\`${b}: **${a}** added, **${m}** modified, **${d}** deleted${extraInfo}`
|
|
}).join('\n')
|
|
|
|
github.rest.issues.createComment({
|
|
issue_number: context.issue.number,
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
body: `## 📸 UI snapshots have been updated
|
|
|
|
**${summaryDiff.total}** snapshot changes in total. **${summaryDiff.added}** added, **${summaryDiff.modified}** modified, **${summaryDiff.deleted}** deleted:
|
|
|
|
${diffByBrowserDisplay}
|
|
|
|
Triggered by [this commit](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.sha }}).
|
|
|
|
👉 **[Review this PR's diff of snapshots.](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/files#:~:text=frontend/__snapshots__/)**`
|
|
})
|