posthog/.github/workflows/benchmark.yml

name: Benchmark

on:
    pull_request:
        branches: ['*']
        paths:
            - .github/workflows/benchmark.yml
    schedule:
        - cron: '0 4 * * 1-5' # Mon-Fri 4AM UTC
    workflow_dispatch: {}

concurrency: 'benchmarks' # Ensure only one of this runs at a time

jobs:
    run-benchmarks:
        name: Clickhouse queries
        runs-on: ubuntu-20.04
        environment: clickhouse-benchmarks

        # Benchmarks are expensive to run so we only run them (periodically) against master branch and for PRs labeled `performance`
        if: ${{ github.repository == 'PostHog/posthog' && (github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'performance'))  }}

        env:
            DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog'
            REDIS_URL: 'redis://localhost'
            DEBUG: '1'
            CLICKHOUSE_DATABASE: posthog
            CLICKHOUSE_HOST: ${{ secrets.BENCHMARKS_CLICKHOUSE_HOST }}
            CLICKHOUSE_USER: ${{ secrets.BENCHMARKS_CLICKHOUSE_USER }}
            CLICKHOUSE_PASSWORD: ${{ secrets.BENCHMARKS_CLICKHOUSE_PASSWORD }}
            CLICKHOUSE_SECURE: 'false'
            CLICKHOUSE_VERIFY: 'false'
            SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only
            BENCHMARK: '1'

        steps:
            - uses: actions/checkout@v3
              with:
                  # Checkout repo with full history
                  fetch-depth: 0

            - name: Check out PostHog/benchmarks-results repo
              uses: actions/checkout@v3
              with:
                  path: ee/benchmarks/results
                  repository: PostHog/benchmark-results
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - name: Stop/Start stack with Docker Compose
              run: |
                  docker compose -f docker-compose.dev.yml down
                  docker compose -f docker-compose.dev.yml up -d                  

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.10.10
                  token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN }}

            - uses: syphar/restore-virtualenv@v1
              id: cache-benchmark-tests

            - uses: syphar/restore-pip-download-cache@v1
              if: steps.cache-benchmark-tests.outputs.cache-hit != 'true'

            - name: Install SAML (python3-saml) dependencies
              shell: bash
              run: |
                  sudo apt-get update
                  sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl                  

            - name: Install python dependencies
              if: steps.cache-benchmark-tests.outputs.cache-hit != 'true'
              run: |
                  python -m pip install -r requirements-dev.txt
                  python -m pip install -r requirements.txt                  

            - name: Install asv
              run: python -m pip install asv==0.5.1 virtualenv

            - name: Set up PostHog
              run: |
                  python manage.py migrate & wait
                  python manage.py setup_dev --no-data                  

            - name: Configure benchmarks
              run: asv machine --config ee/benchmarks/asv.conf.json --yes --machine ci-benchmarks

            - name: Run benchmarks
              run: asv run --config ee/benchmarks/asv.conf.json --show-stderr --strict

            - name: Compare results
              run: |
                  asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 | tee pr_vs_master.txt
                  asv compare $(cat ee/benchmarks/results/last-master-commit) HEAD --config ee/benchmarks/asv.conf.json --factor 1.2 --only-changed | tee pr_vs_master_changed.txt                  

            - name: Save last benchmarked commit
              if: ${{ github.ref == 'refs/heads/master' }}
              run: echo "${{ github.sha }}" | tee ee/benchmarks/results/last-master-commit

            - name: Generate HTML report of results
              if: ${{ github.ref == 'refs/heads/master' }}
              run: asv publish --config ee/benchmarks/asv.conf.json

            - name: Commit update for benchmark results
              if: ${{ github.repository == 'PostHog/posthog' && github.ref == 'refs/heads/master' }}
              uses: stefanzweifel/git-auto-commit-action@v4
              with:
                  repository: ee/benchmarks/results
                  branch: master
                  commit_message: 'Save benchmark results'
                  commit_user_name: PostHog Bot
                  commit_user_email: hey@posthog.com
                  commit_author: PostHog Bot <hey@posthog.com>

            - name: Upload results as artifacts
              uses: actions/upload-artifact@v3
              with:
                  name: benchmarks
                  path: |
                      pr_vs_master.txt
                      pr_vs_master_changed.txt                      

            - name: Read benchmark output
              if: ${{ github.event_name == 'pull_request' }}
              id: pr_vs_master_changed
              uses: juliangruber/read-file-action@v1.0.0
              with:
                  path: pr_vs_master_changed.txt

            - name: Read benchmark output (full)
              if: ${{ github.event_name == 'pull_request' }}
              id: pr_vs_master
              uses: juliangruber/read-file-action@v1.0.0
              with:
                  path: pr_vs_master.txt

            - name: Find Comment
              if: ${{ github.event_name == 'pull_request' }}
              uses: peter-evans/find-comment@v1
              id: fc
              with:
                  issue-number: ${{ github.event.number }}
                  comment-author: 'github-actions[bot]'
                  body-includes: ClickHouse query benchmark results from GitHub Actions

            - name: Create or update comment
              if: ${{ github.event_name == 'pull_request' }}
              uses: peter-evans/create-or-update-comment@v1
              with:
                  comment-id: ${{ steps.fc.outputs.comment-id }}
                  issue-number: ${{ github.event.number }}
                  body: |
                      ClickHouse query benchmark results from GitHub Actions

                      Lower numbers are good, higher numbers are bad. A ratio less than 1
                      means a speed up and greater than 1 means a slowdown. Green lines
                      beginning with `+` are slowdowns (the PR is slower then master or
                      master is slower than the previous release). Red lines beginning
                      with `-` are speedups. Blank means no changes.

                      Significantly changed benchmark results (PR vs master)
                      ```diff
                      ${{ steps.pr_vs_master_changed.outputs.content }}
                      ```

                      <details>
                        <summary>Click to view full benchmark results</summary>

                        ```diff
                        ${{ steps.pr_vs_master.outputs.content }}
                        ```
                      </details>                      
                  edit-mode: replace