posthog/.github/workflows/ci-backend.yml

name: Backend CI

on:
    - pull_request
env:
    SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
    REDIS_URL: 'redis://localhost'

jobs:
    backend-code-quality:
        name: Code quality checks
        runs-on: ubuntu-latest

        services:
            postgres:
                image: postgres:12
                env:
                    POSTGRES_USER: postgres
                    POSTGRES_PASSWORD: postgres
                    POSTGRES_DB: postgres
                ports: ['5432:5432']
                options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

        steps:
            - uses: actions/checkout@v1
              with:
                  fetch-depth: 1

            - name: Set up Python 3.8
              uses: actions/setup-python@v2
              with:
                  python-version: 3.8

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests
              with:
                  requirement_files: requirements.txt # this is optional
            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt

            - name: Check formatting
              run: |
                  black --check .
                  isort --check-only .

            - name: Lint with flake8
              run: |
                  # stop the build if there are Python syntax errors or undefined names
                  flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
                  # exit-zero treats all errors as warnings
                  flake8 . --count --exit-zero --max-complexity=10 --max-line-length=120 --statistics

            - name: Typecheck
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: |
                  mypy .

    django:
        name: Django tests – Py ${{ matrix.python-version }}
        runs-on: ubuntu-latest
        if: ${{ github.actor != 'posthog-contributions-bot[bot]' }}

        strategy:
            fail-fast: false
            matrix:
                python-version: ['3.7.8', '3.8.5', '3.9.0']
        services:
            postgres:
                image: postgres:12
                env:
                    POSTGRES_USER: postgres
                    POSTGRES_PASSWORD: postgres
                    POSTGRES_DB: postgres
                ports: ['5432:5432']
                options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
            clickhouse-server:
                image: yandex/clickhouse-server
                ports:
                    - '8123:8123'
                    - '9000:9000'
                    - '9440:9440'
                    - '9009:9009'
        steps:
            - uses: actions/checkout@v1
              with:
                  fetch-depth: 1

            - name: Set up Python
              uses: actions/setup-python@v2
              with:
                  python-version: ${{ matrix.python-version }}

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests
              with:
                  requirement_files: requirements.txt # this is optional
            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt

            - name: Check migrations
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: python manage.py makemigrations --check --dry-run

            - name: Run posthog tests
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: |
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/shared_dashboard.html
                  pytest posthog/
            - name: Run EE tests
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
                  PRIMARY_DB: 'clickhouse'
                  CLICKHOUSE_HOST: 'localhost'
                  CLICKHOUSE_DATABASE: 'posthog_test'
                  CLICKHOUSE_SECURE: 'False'
                  CLICKHOUSE_VERIFY: 'False'
              run: |
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/shared_dashboard.html
                  pytest ee/

    cloud:
        name: Django tests – Cloud
        runs-on: ubuntu-latest
        if: ${{ github.actor != 'posthog-contributions-bot[bot]' }}

        services:
            postgres:
                image: postgres:12
                env:
                    POSTGRES_USER: postgres
                    POSTGRES_PASSWORD: postgres
                    POSTGRES_DB: postgres
                ports: ['5432:5432']
                options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
            clickhouse-server:
                image: yandex/clickhouse-server
                ports:
                    - '8123:8123'
                    - '9000:9000'
                    - '9440:9440'
                    - '9009:9009'

        steps:
            - name: Fetch posthog-cloud
              run: |
                  curl -L https://github.com/posthog/posthog-cloud/tarball/master | tar --strip-components=1 -xz --
                  mkdir deploy/

            - name: Checkout master
              uses: actions/checkout@v2
              with:
                  ref: 'master'
                  path: 'deploy/'

            - name: Link posthog-cloud at master
              run: |
                  cp -r multi_tenancy deploy/
                  cp -r messaging deploy/
                  cat multi_tenancy_settings.py >> deploy/posthog/settings.py
                  cat requirements.txt >> deploy/requirements.txt

            - name: Set up Python 3.8
              uses: actions/setup-python@v2
              with:
                  python-version: 3.8

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests
              with:
                  requirement_files: requirements.txt # this is optional
            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt

            # The 2-step migration process (first master, then current branch) verifies that it'll always
            # be possible to migrate to the new version without problems in production
            - name: Migrate initially at master, then remove master deploy code
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: |
                  python deploy/manage.py migrate
                  rm -rf deploy

            - name: Checkout current branch
              uses: actions/checkout@v2
              with:
                  path: 'deploy/'

            - name: Install requirements.txt dependencies with pip at current branch
              run: |
                  cd deploy
                  python -m pip install --upgrade pip
                  python -m pip install -r requirements.txt
                  python -m pip install freezegun fakeredis pytest pytest-mock pytest-django

            - name: Link posthog-cloud at current branch
              run: |
                  cp deploy/ee/conftest.py multi_tenancy/conftest.py
                  cp deploy/ee/conftest.py messaging/conftest.py
                  cp -r multi_tenancy deploy/
                  cp -r messaging deploy/
                  cat multi_tenancy_settings.py >> deploy/posthog/settings.py
                  cat requirements.txt >> deploy/requirements.txt

            - name: Check migrations
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: |
                  cd deploy
                  python manage.py makemigrations --check --dry-run
                  python manage.py migrate

            - name: Run posthog tests
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: |
                  cd deploy
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/shared_dashboard.html
                  pytest posthog --reuse-db -m "not skip_on_multitenancy"

            - name: Run cloud tests (posthog-cloud)
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
                  PRIMARY_DB: 'clickhouse'
                  CLICKHOUSE_HOST: 'localhost'
                  CLICKHOUSE_DATABASE: 'posthog_test'
                  CLICKHOUSE_SECURE: 'False'
                  CLICKHOUSE_VERIFY: 'False'
              run: |
                  source .env.template
                  cd deploy
                  pytest multi_tenancy messaging -m "not skip_on_multitenancy"
            - name: Run EE tests
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
                  PRIMARY_DB: 'clickhouse'
                  CLICKHOUSE_HOST: 'localhost'
                  CLICKHOUSE_DATABASE: 'posthog_test'
                  CLICKHOUSE_SECURE: 'False'
                  CLICKHOUSE_VERIFY: 'False'
              run: |
                  cd deploy/
                  pytest ee -s -m "not skip_on_multitenancy"

    foss:
        name: Django tests – FOSS
        runs-on: ubuntu-latest
        if: ${{ github.actor != 'posthog-contributions-bot[bot]' }}

        services:
            postgres:
                image: postgres:12
                env:
                    POSTGRES_USER: postgres
                    POSTGRES_PASSWORD: postgres
                    POSTGRES_DB: postgres
                ports: ['5432:5432']
                options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

        steps:
            - uses: actions/checkout@v1
              with:
                  fetch-depth: 1

            - name: Set up Python 3.8
              uses: actions/setup-python@v2
              with:
                  python-version: 3.8

            - uses: syphar/restore-virtualenv@v1
              id: cache-backend-tests
              with:
                  requirement_files: requirements.txt # this is optional
            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt

            - name: Remove ee
              run: |
                  rm -rf ee/

            - name: Check migrations
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: python manage.py makemigrations --check --dry-run

            - name: Run tests
              env:
                  DATABASE_URL: 'postgres://postgres:postgres@localhost:${{ job.services.postgres.ports[5432] }}/postgres'
              run: |
                  mkdir -p frontend/dist
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/shared_dashboard.html
                  pytest -m "not ee"