mirror of
https://github.com/PostHog/posthog.git
synced 2024-11-28 18:26:15 +01:00
428 lines
17 KiB
YAML
428 lines
17 KiB
YAML
# This workflow runs all of our backend django tests.
|
||
#
|
||
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
|
||
# .github/workflows/ci-backend-update-test-timing.yml action
|
||
name: Backend CI
|
||
|
||
on:
|
||
push:
|
||
branches:
|
||
- master
|
||
pull_request:
|
||
workflow_dispatch:
|
||
inputs:
|
||
clickhouseServerVersion:
|
||
description: ClickHouse server version. Leave blank for default
|
||
type: string
|
||
|
||
env:
|
||
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
REDIS_URL: 'redis://localhost'
|
||
CLICKHOUSE_HOST: 'localhost'
|
||
CLICKHOUSE_SECURE: 'False'
|
||
CLICKHOUSE_VERIFY: 'False'
|
||
TEST: 1
|
||
CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}
|
||
OBJECT_STORAGE_ENABLED: 'True'
|
||
OBJECT_STORAGE_ENDPOINT: 'http://localhost:19000'
|
||
OBJECT_STORAGE_ACCESS_KEY_ID: 'object_storage_root_user'
|
||
OBJECT_STORAGE_SECRET_ACCESS_KEY: 'object_storage_root_password'
|
||
|
||
concurrency:
|
||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||
cancel-in-progress: true
|
||
|
||
jobs:
|
||
# Job to decide if we should run backend ci
|
||
# See https://github.com/dorny/paths-filter#conditional-execution for more details
|
||
changes:
|
||
runs-on: ubuntu-latest
|
||
timeout-minutes: 5
|
||
if: github.repository == 'PostHog/posthog'
|
||
name: Determine need to run backend checks
|
||
# Set job outputs to values from filter step
|
||
outputs:
|
||
backend: ${{ steps.filter.outputs.backend }}
|
||
steps:
|
||
# For pull requests it's not necessary to checkout the code, but we
|
||
# also want this to run on master so we need to checkout
|
||
- uses: actions/checkout@v3
|
||
|
||
- uses: dorny/paths-filter@v2
|
||
id: filter
|
||
with:
|
||
filters: |
|
||
backend:
|
||
# Avoid running backend tests for irrelevant changes
|
||
# NOTE: we are at risk of missing a dependency here. We could make
|
||
# the dependencies more clear if we separated the backend/frontend
|
||
# code completely
|
||
- 'ee/**/*'
|
||
- 'posthog/**/*'
|
||
- 'bin/*.py'
|
||
- requirements.txt
|
||
- requirements-dev.txt
|
||
- mypy.ini
|
||
- pytest.ini
|
||
# Make sure we run if someone is explicitly change the workflow
|
||
- .github/workflows/ci-backend.yml
|
||
- .github/workflows/backend-tests-action/action.yml
|
||
# We use docker compose for tests, make sure we rerun on
|
||
# changes to docker-compose.dev.yml e.g. dependency
|
||
# version changes
|
||
- docker-compose.dev.yml
|
||
|
||
backend-code-quality:
|
||
needs: changes
|
||
timeout-minutes: 5
|
||
# Make sure we only run on backend changes
|
||
if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' }}
|
||
|
||
name: Python code quality checks
|
||
runs-on: ubuntu-latest
|
||
|
||
steps:
|
||
- uses: actions/checkout@v3
|
||
with:
|
||
fetch-depth: 1
|
||
|
||
- name: Stop/Start stack with Docker Compose
|
||
run: |
|
||
docker compose -f docker-compose.dev.yml down
|
||
docker compose -f docker-compose.dev.yml up -d
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v4
|
||
with:
|
||
python-version: 3.8.14
|
||
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
with:
|
||
custom_cache_key_element: v1-
|
||
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r requirements-dev.txt
|
||
python -m pip install -r requirements.txt
|
||
|
||
- name: Check formatting
|
||
run: |
|
||
black --check .
|
||
isort --check-only .
|
||
|
||
- name: Check for errors and code style violations
|
||
run: |
|
||
flake8 .
|
||
|
||
- name: Check static typing
|
||
run: |
|
||
mypy .
|
||
|
||
check-migrations:
|
||
needs: changes
|
||
timeout-minutes: 5
|
||
|
||
if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' }}
|
||
|
||
name: Validate Django migrations
|
||
runs-on: ubuntu-latest
|
||
|
||
steps:
|
||
- uses: actions/checkout@v3
|
||
|
||
- name: Stop/Start stack with Docker Compose
|
||
run: |
|
||
docker compose -f docker-compose.dev.yml down
|
||
docker compose -f docker-compose.dev.yml up -d
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v4
|
||
with:
|
||
python-version: 3.8.14
|
||
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
with:
|
||
custom_cache_key_element: v1-
|
||
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r requirements-dev.txt
|
||
python -m pip install -r requirements.txt
|
||
|
||
- uses: actions/checkout@v3
|
||
with:
|
||
ref: master
|
||
|
||
- name: Run migrations up to master
|
||
run: |
|
||
python manage.py migrate
|
||
|
||
- uses: actions/checkout@v3
|
||
|
||
- name: Check migrations
|
||
run: |
|
||
python manage.py makemigrations --check --dry-run
|
||
git fetch origin master
|
||
# `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
|
||
# With `--name-status` we get the same, but including a column for status, respectively: A, D, M
|
||
# In this check we exclusively care about files that were added (A) in posthog/migrations/
|
||
git diff --name-status origin/master..HEAD | grep "A\sposthog/migrations/" | awk '{print $2}' | python manage.py test_migrations_are_safe
|
||
|
||
django:
|
||
needs: changes
|
||
timeout-minutes: 30
|
||
if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' || github.event_name == 'workflow_dispatch' }}
|
||
|
||
name: Django tests – ${{ matrix.segment }} (persons-on-events ${{ matrix.person-on-events && 'on' || 'off' }}), Py ${{ matrix.python-version }}, ${{ matrix.clickhouse-server-image }} (${{matrix.group}}/${{ matrix.concurrency }})
|
||
runs-on: ubuntu-latest
|
||
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
python-version: ['3.8.14']
|
||
clickhouse-server-image: ['clickhouse/clickhouse-server:22.3']
|
||
segment: ['FOSS', 'EE']
|
||
person-on-events: [false, true]
|
||
# :NOTE: Keep concurrency and groups in sync
|
||
concurrency: [5]
|
||
group: [1, 2, 3, 4, 5]
|
||
|
||
steps:
|
||
- uses: actions/checkout@v3
|
||
with:
|
||
fetch-depth: 1
|
||
repository: ${{ github.event.pull_request.head.repo.full_name }}
|
||
ref: ${{ github.event.pull_request.head.ref }}
|
||
# Use PostHog Bot token when not on forks to enable proper snapshot updating
|
||
token: ${{ github.event.pull_request.head.repo.full_name == github.repository && secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }}
|
||
|
||
- uses: ./.github/actions/run-backend-tests
|
||
with:
|
||
segment: ${{ matrix.segment }}
|
||
person-on-events: ${{ matrix.person-on-events }}
|
||
python-version: ${{ matrix.python-version }}
|
||
clickhouse-server-image: ${{ matrix.clickhouse-server-image }}
|
||
concurrency: ${{ matrix.concurrency }}
|
||
group: ${{ matrix.group }}
|
||
|
||
- uses: EndBug/add-and-commit@v9
|
||
# Skip on forks
|
||
# Also skip for persons-on-events runs, as we want to ignore snapshots diverging there
|
||
if: ${{ !matrix.person-on-events && github.event.pull_request.head.repo.full_name == github.repository }}
|
||
with:
|
||
add: '["ee", "posthog/api/test/__snapshots__", "posthog/test/__snapshots__", "posthog/queries/"]'
|
||
message: 'Update snapshots'
|
||
default_author: github_actions
|
||
|
||
- name: Check if any snapshot changes were left uncomitted
|
||
id: changed-files
|
||
if: ${{ !matrix.person-on-events && github.event.pull_request.head.repo.full_name == github.repository }}
|
||
run: |
|
||
if [[ -z $(git status -s | grep -v ".test_durations" | tr -d "\n") ]]
|
||
then
|
||
echo 'files_found=false' >> $GITHUB_OUTPUT
|
||
else
|
||
echo 'diff=$(git status --porcelain)' >> $GITHUB_OUTPUT
|
||
echo 'files_found=true' >> $GITHUB_OUTPUT
|
||
fi
|
||
|
||
- name: Fail CI if some snapshots have been updated but not committed
|
||
if: steps.changed-files.outputs.files_found == 'true' && steps.add-and-commit.outcome == 'success'
|
||
run: |
|
||
echo "${{ steps.changed-files.outputs.diff }}"
|
||
exit 1
|
||
|
||
- name: Archive email renders
|
||
uses: actions/upload-artifact@v3
|
||
if: matrix.foss
|
||
with:
|
||
name: email_renders
|
||
path: posthog/tasks/test/__emails__
|
||
|
||
cloud:
|
||
needs: changes
|
||
timeout-minutes: 30
|
||
if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' }}
|
||
|
||
name: Django tests – Cloud
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Fetch posthog-cloud
|
||
run: |
|
||
curl -u posthog-bot:${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }} -L https://github.com/posthog/posthog-cloud/tarball/master | tar --strip-components=1 -xz --
|
||
|
||
- name: Checkout master
|
||
uses: actions/checkout@v3
|
||
with:
|
||
ref: 'master'
|
||
path: 'master/'
|
||
|
||
- name: Link posthog-cloud at master
|
||
run: |
|
||
cp -r multi_tenancy master/
|
||
cp -r messaging master/
|
||
cat multi_tenancy_settings.py > master/posthog/settings/cloud.py
|
||
cat requirements.txt >> master/requirements.txt
|
||
|
||
- name: Stop/Start stack with Docker Compose
|
||
run: |
|
||
docker compose -f master/docker-compose.dev.yml down
|
||
docker compose -f master/docker-compose.dev.yml up -d
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v4
|
||
with:
|
||
python-version: 3.8.14
|
||
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r master/requirements-dev.txt
|
||
python -m pip install -r master/requirements.txt
|
||
|
||
- name: Wait for Clickhouse & Kafka
|
||
run: master/bin/check_kafka_clickhouse_up
|
||
|
||
# The 2-step migration process (first master, then current branch) verifies that it'll always
|
||
# be possible to migrate to the new version without problems in production
|
||
- name: Run migration on master branch
|
||
run: |
|
||
python master/manage.py migrate
|
||
|
||
- name: Checkout current branch
|
||
uses: actions/checkout@v3
|
||
with:
|
||
path: 'current/'
|
||
|
||
- name: Install requirements.txt dependencies with pip at current branch
|
||
run: |
|
||
cd current
|
||
python -m pip install --upgrade pip
|
||
python -m pip install -r requirements.txt
|
||
python -m pip install freezegun fakeredis pytest pytest-mock pytest-django syrupy
|
||
|
||
- name: Link posthog-cloud at current branch
|
||
run: |
|
||
cp current/ee/conftest.py multi_tenancy/conftest.py
|
||
cp current/ee/conftest.py messaging/conftest.py
|
||
cp -r multi_tenancy current/
|
||
cp -r messaging current/
|
||
cat multi_tenancy_settings.py > current/posthog/settings/cloud.py
|
||
cat requirements.txt >> current/requirements.txt
|
||
|
||
- name: Check migrations
|
||
run: |
|
||
cd current
|
||
python manage.py makemigrations --check --dry-run
|
||
python manage.py migrate
|
||
|
||
- name: Set up needed files
|
||
run: |
|
||
cd current
|
||
mkdir -p frontend/dist
|
||
python manage.py collectstatic --noinput
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/exporter.html
|
||
|
||
- name: Run cloud tests (posthog-cloud)
|
||
run: |
|
||
source .env.template
|
||
cd current
|
||
pytest multi_tenancy messaging -m "not skip_on_multitenancy and not async_migrations" --durations=100 --durations-min=1.0
|
||
|
||
async-migrations:
|
||
name: Async migrations tests
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: 'Checkout repo'
|
||
uses: actions/checkout@v3
|
||
with:
|
||
fetch-depth: 1
|
||
|
||
- name: Start stack with Docker Compose
|
||
shell: bash
|
||
run: |
|
||
export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ inputs.clickhouse-server-image-version }}
|
||
docker compose -f docker-compose.dev.yml down
|
||
docker compose -f docker-compose.dev.yml up -d
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v4
|
||
with:
|
||
python-version: 3.8.14
|
||
|
||
- name: Install SAML (python3-saml) dependencies
|
||
shell: bash
|
||
run: |
|
||
sudo apt-get update
|
||
sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl
|
||
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-async-migrations-tests
|
||
with:
|
||
custom_cache_key_element: v1-${{ inputs.cache-id }}
|
||
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
shell: bash
|
||
run: |
|
||
python -m pip install -r requirements-dev.txt
|
||
python -m pip install -r requirements.txt
|
||
|
||
- name: Add kafka host to /etc/hosts for kafka connectivity
|
||
shell: bash
|
||
run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts
|
||
|
||
- name: Set up needed files
|
||
shell: bash
|
||
run: |
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/exporter.html
|
||
|
||
- name: Wait for Clickhouse & Kafka
|
||
shell: bash
|
||
run: bin/check_kafka_clickhouse_up
|
||
|
||
- name: Run async migrations tests
|
||
shell: bash
|
||
run: |
|
||
pytest -m "async_migrations"
|