posthog/.github/workflows/ci-backend.yml

# This workflow runs all of our backend django tests.
#
# If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching
# .github/workflows/ci-backend-update-test-timing.yml action
name: Backend CI

on:
    push:
        branches:
            - master
    pull_request:
    workflow_dispatch:
        inputs:
            clickhouseServerVersion:
                description: ClickHouse server version. Leave blank for default
                type: string

env:
    SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
    DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
    REDIS_URL: 'redis://localhost'
    CLICKHOUSE_HOST: 'localhost'
    CLICKHOUSE_SECURE: 'False'
    CLICKHOUSE_VERIFY: 'False'
    SAML_DISABLED: 1
    TEST: 1
    CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }}

jobs:
    # Job to decide if we should run backend ci
    # See https://github.com/dorny/paths-filter#conditional-execution for more details
    changes:
        runs-on: ubuntu-latest
        if: github.repository == 'PostHog/posthog'
        name: Determine need to run backend checks
        # Set job outputs to values from filter step
        outputs:
            backend: ${{ steps.filter.outputs.backend }}
        steps:
            # For pull requests it's not necessary to checkout the code, but we
            # also want this to run on master so we need to checkout
            - uses: actions/checkout@v2

            - uses: dorny/paths-filter@v2
              id: filter
              with:
                  filters: |
                      backend:
                        # Avoid running backend tests for irrelevant changes
                        # NOTE: we are at risk of missing a dependency here. We could make
                        # the dependencies more clear if we separated the backend/frontend
                        # code completely
                        - 'ee/**/*'
                        - 'posthog/**/*'
                        - requirements.txt
                        - requirements-dev.txt
                        - mypy.ini
                        - pytest.ini
                        # Make sure we run if someone is explicitly change the workflow
                        - .github/workflows/ci-backend.yml
                        - .github/workflows/backend-tests-action/action.yml

    backend-code-quality:
        needs: changes
        # Make sure we only run on backend changes
        if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' }}

        name: Python code quality checks
        runs-on: ubuntu-latest

        services:
            postgres:
                image: postgres:12
                env:
                    POSTGRES_USER: posthog
                    POSTGRES_PASSWORD: posthog
                    POSTGRES_DB: posthog
                ports: ['5432:5432']
                options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5

        steps:
            - uses: actions/checkout@v2
              with:
                  fetch-depth: 1

            - name: Set up Python 3.8
              uses: actions/setup-python@v2
              with:
                  python-version: 3.8

            - uses: syphar/restore-virtualenv@v1.2
              id: cache-backend-tests
              with:
                  custom_cache_key_element: v1-

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt

            - name: Check formatting
              run: |
                  black --check .
                  isort --check-only .

            - name: Check for errors and code style violations
              run: |
                  flake8 .

            - name: Check static typing
              run: |
                  mypy .

    check-migrations:
        needs: changes
        if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' }}

        name: Validate django migrations
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v2

            - name: Start stack with Docker Compose
              run: |
                  docker-compose -f docker-compose.dev.yml down
                  docker-compose -f docker-compose.dev.yml up -d db &

            - name: Set up Python
              uses: actions/setup-python@v2
              with:
                  python-version: 3.8.5

            - uses: syphar/restore-virtualenv@v1.2
              id: cache-backend-tests
              with:
                  custom_cache_key_element: v1-

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt

            - name: Check migrations
              run: |
                  python manage.py makemigrations --check --dry-run
                  git fetch origin master
                  # `git diff --name-only` returns a list of files that were changed - added OR deleted OR modified
                  # With `--name-status` we get the same, but including a column for status, respectively: A, D, M
                  # In this check we exclusively care about files that were added (A) in posthog/migrations/
                  git diff --name-status origin/master..HEAD | grep "A\tposthog/migrations/" | awk '{print $2}' | python manage.py test_migrations_are_null

    django:
        needs: changes
        if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' || github.event_name == 'workflow_dispatch' }}

        name: Django tests – Py ${{ matrix.python-version }} ${{ matrix.name }} (${{matrix.group}}/${{ matrix.concurrency }})
        runs-on: ubuntu-latest

        strategy:
            fail-fast: false
            matrix:
                python-version: ['3.8.5']
                ee: [true]
                foss: [false]
                saml: [false]
                name: ['']
                # :NOTE: Keep concurrency and group's in sync
                concurrency: [5]
                group: [1, 2, 3, 4, 5]
                include:
                    # :TRICKY: Run FOSS tests in a separate container
                    - python-version: '3.8.5'
                      ee: false
                      saml: false
                      foss: true
                      name: 'FOSS'
                      concurrency: 1
                      group: 1
                    # :TRICKY: Run FOSS tests in a separate container
                    - python-version: '3.9.0'
                      ee: false
                      saml: false
                      foss: true
                      name: 'FOSS'
                      concurrency: 1
                      group: 1
                    # :TRICKY: Run SAML tests in a separate container
                    - python-version: '3.8.5'
                      ee: false
                      saml: true
                      foss: false
                      name: 'SAML'
                      concurrency: 1
                      group: 1
        steps:
            - uses: actions/checkout@v2
              with:
                  fetch-depth: 1

            - uses: ./.github/actions/run-backend-tests
              with:
                  cache-id: ${{ matrix.name }}
                  python-version: ${{ matrix.python-version }}
                  ee: ${{ matrix.ee }}
                  foss: ${{ matrix.foss }}
                  saml: ${{ matrix.saml }}
                  concurrency: ${{ matrix.concurrency }}
                  group: ${{ matrix.group }}

    cloud:
        needs: changes
        if: ${{ needs.changes.outputs.backend == 'true' && github.repository == 'PostHog/posthog' }}

        name: Django tests – Cloud
        runs-on: ubuntu-latest
        steps:
            - name: Fetch posthog-cloud
              run: |
                  curl -L https://github.com/posthog/posthog-cloud/tarball/master | tar --strip-components=1 -xz --
                  mkdir deploy/
            - name: Checkout master
              uses: actions/checkout@v2
              with:
                  ref: 'master'
                  path: 'deploy/'
            - name: Link posthog-cloud at master
              run: |
                  cp -r multi_tenancy deploy/
                  cp -r messaging deploy/
                  cat multi_tenancy_settings.py > deploy/posthog/settings/cloud.py
                  cat requirements.txt >> deploy/requirements.txt
            - name: Start stack with Docker Compose
              run: |
                  docker-compose -f deploy/docker-compose.dev.yml down
                  docker-compose -f deploy/docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis &
            - name: Set up Python 3.8
              uses: actions/setup-python@v2
              with:
                  python-version: 3.8

            - uses: syphar/restore-virtualenv@v1.2
              id: cache-backend-tests

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'

            - name: Install python dependencies
              if: steps.cache-backend-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r deploy/requirements-dev.txt
                  python -m pip install -r deploy/requirements.txt

            - name: Wait for Clickhouse & Kafka
              run: deploy/bin/check_kafka_clickhouse_up

            # The 2-step migration process (first master, then current branch) verifies that it'll always
            # be possible to migrate to the new version without problems in production
            - name: Migrate initially at master, then remove master deploy code
              run: |
                  python deploy/manage.py migrate
                  rm -rf deploy

            - name: Checkout current branch
              uses: actions/checkout@v2
              with:
                  path: 'deploy/'

            - name: Install requirements.txt dependencies with pip at current branch
              run: |
                  cd deploy
                  python -m pip install --upgrade pip
                  python -m pip install -r requirements.txt
                  python -m pip install freezegun fakeredis pytest pytest-mock pytest-django syrupy

            - name: Link posthog-cloud at current branch
              run: |
                  cp deploy/ee/conftest.py multi_tenancy/conftest.py
                  cp deploy/ee/conftest.py messaging/conftest.py
                  cp -r multi_tenancy deploy/
                  cp -r messaging deploy/
                  cat multi_tenancy_settings.py > deploy/posthog/settings/cloud.py
                  cat requirements.txt >> deploy/requirements.txt

            - name: Check migrations
              run: |
                  cd deploy
                  python manage.py makemigrations --check --dry-run
                  python manage.py migrate

            - name: Add kafka host to /etc/hosts for kafka connectivity
              run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts

            - name: Set up needed files
              run: |
                  cd deploy
                  mkdir -p frontend/dist
                  python manage.py collectstatic --noinput
                  touch frontend/dist/index.html
                  touch frontend/dist/layout.html
                  touch frontend/dist/shared_dashboard.html

            - name: Run cloud tests (posthog-cloud)
              run: |
                  source .env.template
                  cd deploy
                  pytest multi_tenancy messaging -m "not skip_on_multitenancy"