mirror of
https://github.com/PostHog/posthog.git
synced 2024-11-30 19:41:46 +01:00
751a35cd35
* Make DDLs more friendly towards running on a cluster
* Use primary CLICKHOUSE host for migrations and DDL
* loose ends on person kafka create
* posthog -> cluster typo
* add cluster to KAFKA create for plugin logs
* Feed the type monster
* clusterfy local clickhouse
* test docker-compose backed github action
* run just clickhouse and postgres from docker-compose
* move option to between up and <services>
* posthog all the things
* suggest tests run on cluster
* posthog cluster for ci
* use deploy path for docker-compose
* fix for a clickhouse bug 🐛
* complete CH bug fixes
* 5439 the github actions pg configs
* remove CLICKHOUSE_DATABASE (handled automatically)
* update DATABASE_URL for code quality checks
* Missed a few DDLs on Person
* 5439 -> 5432 to please the people
* cleanup persons and use f strings <3 f strings
* remove auto parens
* Update requirements to use our fork of infi.clickhouse_orm
* fix person.py formatting
* Include boilerplate macros for a cluster
279 lines
11 KiB
YAML
279 lines
11 KiB
YAML
name: Backend CI
|
||
|
||
on:
|
||
- pull_request
|
||
env:
|
||
SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
|
||
REDIS_URL: 'redis://localhost'
|
||
|
||
jobs:
|
||
backend-code-quality:
|
||
name: Code quality checks
|
||
runs-on: ubuntu-latest
|
||
|
||
services:
|
||
postgres:
|
||
image: postgres:12
|
||
env:
|
||
POSTGRES_USER: posthog
|
||
POSTGRES_PASSWORD: posthog
|
||
POSTGRES_DB: posthog
|
||
ports: ['5432:5432']
|
||
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||
|
||
steps:
|
||
- uses: actions/checkout@v1
|
||
with:
|
||
fetch-depth: 1
|
||
|
||
- name: Set up Python 3.8
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: 3.8
|
||
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r requirements-dev.txt
|
||
python -m pip install -r requirements.txt
|
||
|
||
- name: Check formatting
|
||
run: |
|
||
black --check .
|
||
isort --check-only .
|
||
|
||
- name: Lint with flake8
|
||
run: |
|
||
# stop the build if there are Python syntax errors or undefined names
|
||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||
# exit-zero treats all errors as warnings
|
||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics
|
||
|
||
- name: Typecheck
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
run: |
|
||
mypy .
|
||
|
||
django:
|
||
name: Django tests – Py ${{ matrix.python-version }}
|
||
runs-on: ubuntu-latest
|
||
|
||
strategy:
|
||
fail-fast: false
|
||
matrix:
|
||
python-version: ['3.7.8', '3.8.5', '3.9.0']
|
||
steps:
|
||
- uses: actions/checkout@v1
|
||
with:
|
||
fetch-depth: 1
|
||
- name: Start stack with Docker Compose
|
||
run: docker-compose -f ee/docker-compose.ch.yml up -d db clickhouse
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: ${{ matrix.python-version }}
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r requirements-dev.txt
|
||
python -m pip install -r requirements.txt
|
||
|
||
- name: Check migrations
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
run: python manage.py makemigrations --check --dry-run
|
||
|
||
- name: Run posthog tests
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
run: |
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/shared_dashboard.html
|
||
pytest posthog/
|
||
- name: Run EE tests
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
PRIMARY_DB: 'clickhouse'
|
||
CLICKHOUSE_HOST: 'localhost'
|
||
CLICKHOUSE_CLUSTER: 'posthog'
|
||
CLICKHOUSE_SECURE: 'False'
|
||
CLICKHOUSE_VERIFY: 'False'
|
||
run: |
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/shared_dashboard.html
|
||
pytest ee/
|
||
|
||
cloud:
|
||
name: Django tests – Cloud
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- name: Fetch posthog-cloud
|
||
run: |
|
||
curl -L https://github.com/posthog/posthog-cloud/tarball/master | tar --strip-components=1 -xz --
|
||
mkdir deploy/
|
||
- name: Checkout master
|
||
uses: actions/checkout@v2
|
||
with:
|
||
ref: 'master'
|
||
path: 'deploy/'
|
||
- name: Link posthog-cloud at master
|
||
run: |
|
||
cp -r multi_tenancy deploy/
|
||
cp -r messaging deploy/
|
||
cat multi_tenancy_settings.py >> deploy/posthog/settings.py
|
||
cat requirements.txt >> deploy/requirements.txt
|
||
- name: Start stack with Docker Compose
|
||
run: docker-compose -f deploy/ee/docker-compose.ch.yml up -d db clickhouse
|
||
- name: Set up Python 3.8
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: 3.8
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r deploy/requirements-dev.txt
|
||
python -m pip install -r deploy/requirements.txt
|
||
|
||
# The 2-step migration process (first master, then current branch) verifies that it'll always
|
||
# be possible to migrate to the new version without problems in production
|
||
- name: Migrate initially at master, then remove master deploy code
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
run: |
|
||
python deploy/manage.py migrate
|
||
rm -rf deploy
|
||
|
||
- name: Checkout current branch
|
||
uses: actions/checkout@v2
|
||
with:
|
||
path: 'deploy/'
|
||
|
||
- name: Install requirements.txt dependencies with pip at current branch
|
||
run: |
|
||
cd deploy
|
||
python -m pip install --upgrade pip
|
||
python -m pip install -r requirements.txt
|
||
python -m pip install freezegun fakeredis pytest pytest-mock pytest-django
|
||
|
||
- name: Link posthog-cloud at current branch
|
||
run: |
|
||
cp deploy/ee/conftest.py multi_tenancy/conftest.py
|
||
cp deploy/ee/conftest.py messaging/conftest.py
|
||
cp -r multi_tenancy deploy/
|
||
cp -r messaging deploy/
|
||
cat multi_tenancy_settings.py >> deploy/posthog/settings.py
|
||
cat requirements.txt >> deploy/requirements.txt
|
||
|
||
- name: Check migrations
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
run: |
|
||
cd deploy
|
||
python manage.py makemigrations --check --dry-run
|
||
python manage.py migrate
|
||
|
||
- name: Run posthog tests
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
run: |
|
||
cd deploy
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/shared_dashboard.html
|
||
pytest posthog --reuse-db -m "not skip_on_multitenancy"
|
||
|
||
- name: Run cloud tests (posthog-cloud)
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
PRIMARY_DB: 'clickhouse'
|
||
CLICKHOUSE_HOST: 'localhost'
|
||
CLICKHOUSE_CLUSTER: 'posthog'
|
||
CLICKHOUSE_SECURE: 'False'
|
||
CLICKHOUSE_VERIFY: 'False'
|
||
run: |
|
||
source .env.template
|
||
cd deploy
|
||
pytest multi_tenancy messaging -m "not skip_on_multitenancy"
|
||
- name: Run EE tests
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
|
||
PRIMARY_DB: 'clickhouse'
|
||
CLICKHOUSE_HOST: 'localhost'
|
||
CLICKHOUSE_CLUSTER: 'posthog'
|
||
CLICKHOUSE_SECURE: 'False'
|
||
CLICKHOUSE_VERIFY: 'False'
|
||
run: |
|
||
cd deploy/
|
||
pytest ee -s -m "not skip_on_multitenancy"
|
||
|
||
foss:
|
||
name: Django tests – FOSS
|
||
runs-on: ubuntu-latest
|
||
|
||
services:
|
||
postgres:
|
||
image: postgres:12
|
||
env:
|
||
POSTGRES_USER: posthog
|
||
POSTGRES_PASSWORD: posthog
|
||
POSTGRES_DB: posthog
|
||
ports: ['5432:5432']
|
||
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
|
||
|
||
steps:
|
||
- uses: actions/checkout@v1
|
||
with:
|
||
fetch-depth: 1
|
||
|
||
- name: Set up Python 3.8
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: 3.8
|
||
|
||
- uses: syphar/restore-virtualenv@v1
|
||
id: cache-backend-tests
|
||
- uses: syphar/restore-pip-download-cache@v1
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
- name: Install python dependencies
|
||
if: steps.cache-backend-tests.outputs.cache-hit != 'true'
|
||
run: |
|
||
python -m pip install -r requirements-dev.txt
|
||
python -m pip install -r requirements.txt
|
||
|
||
- name: Remove ee
|
||
run: |
|
||
rm -rf ee/
|
||
|
||
- name: Check migrations
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:${{ job.services.postgres.ports[5432] }}/posthog'
|
||
run: python manage.py makemigrations --check --dry-run
|
||
|
||
- name: Run tests
|
||
env:
|
||
DATABASE_URL: 'postgres://posthog:posthog@localhost:${{ job.services.postgres.ports[5432] }}/posthog'
|
||
run: |
|
||
mkdir -p frontend/dist
|
||
touch frontend/dist/index.html
|
||
touch frontend/dist/layout.html
|
||
touch frontend/dist/shared_dashboard.html
|
||
pytest -m "not ee"
|