From 4f41076aca8f44075b660159cb9ce29c95df81ff Mon Sep 17 00:00:00 2001 From: Yakko Majuri <38760734+yakkomajuri@users.noreply.github.com> Date: Tue, 26 Apr 2022 16:58:30 +0000 Subject: [PATCH] chore: split out async migrations ci (#9539) * chore: split out async migrations ci * update * mark test correctly * remove if from runner --- .github/actions/run-backend-tests/action.yml | 2 +- .github/workflows/ci-async-migrations.yml | 83 +++++++++++++++++++ .github/workflows/ci-backend.yml | 2 +- .../test/test_0002_events_sample_by.py | 2 +- .../test_0003_fill_person_distinct_id2.py | 2 + .../test/test_0004_replicated_schema.py | 4 +- pytest.ini | 3 +- 7 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/ci-async-migrations.yml diff --git a/.github/actions/run-backend-tests/action.yml b/.github/actions/run-backend-tests/action.yml index 5bb24905fd0..5435d29d13c 100644 --- a/.github/actions/run-backend-tests/action.yml +++ b/.github/actions/run-backend-tests/action.yml @@ -82,7 +82,7 @@ runs: if: ${{ inputs.foss == 'true' }} shell: bash run: | - pytest -m "not ee" posthog/ --cov --cov-report=xml:coverage-postgres.xml + pytest -m "not ee and not async_migrations" posthog/ --cov --cov-report=xml:coverage-postgres.xml - name: Run ee/ tests if: ${{ inputs.ee == 'true' }} diff --git a/.github/workflows/ci-async-migrations.yml b/.github/workflows/ci-async-migrations.yml new file mode 100644 index 00000000000..ab9042e499b --- /dev/null +++ b/.github/workflows/ci-async-migrations.yml @@ -0,0 +1,83 @@ +name: Async Migrations CI + +on: + pull_request: + paths: + - 'posthog/async_migrations/**' + - 'ee/clickhouse/sql/**' + - 'ee/clickhouse/migrations/**' + - 'ee/migrations/**' + +env: + SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' # unsafe - for testing only + DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' + REDIS_URL: 'redis://localhost' + CLICKHOUSE_HOST: 'localhost' + CLICKHOUSE_SECURE: 'False' + CLICKHOUSE_VERIFY: 'False' + TEST: 1 + CLICKHOUSE_SERVER_IMAGE_VERSION: ${{ github.event.inputs.clickhouseServerVersion || '' }} + +jobs: + async-migrations: + name: Async Migrations tests + runs-on: ubuntu-latest + steps: + - name: 'Checkout repo' + uses: actions/checkout@v2 + with: + fetch-depth: 1 + + - name: Start stack with Docker Compose + shell: bash + run: | + export CLICKHOUSE_SERVER_IMAGE_VERSION=${{ inputs.clickhouse-server-image-version }} + docker-compose -f docker-compose.dev.yml down + docker-compose -f docker-compose.dev.yml up -d db clickhouse zookeeper kafka redis & + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ inputs.python-version }} + + - name: Install SAML (python3-saml) dependencies + shell: bash + run: | + sudo apt-get update + sudo apt-get install libxml2-dev libxmlsec1-dev libxmlsec1-openssl + + - uses: syphar/restore-virtualenv@v1.2 + id: cache-async-migrations-tests + with: + custom_cache_key_element: v1-${{ inputs.cache-id }} + + - uses: syphar/restore-pip-download-cache@v1 + if: steps.cache-backend-tests.outputs.cache-hit != 'true' + + - name: Install python dependencies + if: steps.cache-backend-tests.outputs.cache-hit != 'true' + shell: bash + run: | + python -m pip install -r requirements-dev.txt + python -m pip install -r requirements.txt + + - name: Add kafka host to /etc/hosts for kafka connectivity + shell: bash + run: sudo echo "127.0.0.1 kafka" | sudo tee -a /etc/hosts + + - name: Set up needed files + shell: bash + run: | + mkdir -p frontend/dist + touch frontend/dist/index.html + touch frontend/dist/layout.html + touch frontend/dist/shared_dashboard.html + + - name: Wait for Clickhouse & Kafka + shell: bash + run: bin/check_kafka_clickhouse_up + + - name: Run async migrations tests + shell: bash + run: | + pytest -m "async_migrations" posthog/ --cov --cov-report=xml:coverage-postgres.xml diff --git a/.github/workflows/ci-backend.yml b/.github/workflows/ci-backend.yml index fbf4c4970e1..0a859b373d2 100644 --- a/.github/workflows/ci-backend.yml +++ b/.github/workflows/ci-backend.yml @@ -313,4 +313,4 @@ jobs: run: | source .env.template cd deploy - pytest multi_tenancy messaging -m "not skip_on_multitenancy" + pytest multi_tenancy messaging -m "not skip_on_multitenancy and not async_migrations" diff --git a/posthog/async_migrations/test/test_0002_events_sample_by.py b/posthog/async_migrations/test/test_0002_events_sample_by.py index 2d0a135947e..8b68a62f028 100644 --- a/posthog/async_migrations/test/test_0002_events_sample_by.py +++ b/posthog/async_migrations/test/test_0002_events_sample_by.py @@ -87,7 +87,7 @@ class Test0002EventsSampleBy(AsyncMigrationBaseTest): execute_query(f"DROP TABLE {CLICKHOUSE_DATABASE}.events") # Run the full migration through - @pytest.mark.ee + @pytest.mark.async_migrations def test_run_migration_in_full(self): from posthog.client import sync_execute diff --git a/posthog/async_migrations/test/test_0003_fill_person_distinct_id2.py b/posthog/async_migrations/test/test_0003_fill_person_distinct_id2.py index d9fb77dbe08..bbe5a112b8f 100644 --- a/posthog/async_migrations/test/test_0003_fill_person_distinct_id2.py +++ b/posthog/async_migrations/test/test_0003_fill_person_distinct_id2.py @@ -21,6 +21,7 @@ class Test0003FillPersonDistinctId2(AsyncMigrationBaseTest): sync_execute("TRUNCATE TABLE person_distinct_id2") sync_execute("ALTER TABLE person_distinct_id COMMENT COLUMN distinct_id 'dont_skip_0003'") + @pytest.mark.async_migrations def test_is_required(self): from posthog.client import sync_execute @@ -29,6 +30,7 @@ class Test0003FillPersonDistinctId2(AsyncMigrationBaseTest): sync_execute("ALTER TABLE person_distinct_id COMMENT COLUMN distinct_id 'skip_0003_fill_person_distinct_id2'") self.assertFalse(self.migration.is_required()) + @pytest.mark.async_migrations def test_migration(self): from posthog.client import sync_execute diff --git a/posthog/async_migrations/test/test_0004_replicated_schema.py b/posthog/async_migrations/test/test_0004_replicated_schema.py index 23091047b7c..d8d56ccb6f4 100644 --- a/posthog/async_migrations/test/test_0004_replicated_schema.py +++ b/posthog/async_migrations/test/test_0004_replicated_schema.py @@ -28,7 +28,6 @@ def _create_event(**kwargs): create_event(**kwargs) -@pytest.mark.ee class Test0004ReplicatedSchema(AsyncMigrationBaseTest, ClickhouseTestMixin): def setUp(self): self.recreate_database() @@ -49,6 +48,7 @@ class Test0004ReplicatedSchema(AsyncMigrationBaseTest, ClickhouseTestMixin): sync_execute(f"CREATE DATABASE {settings.CLICKHOUSE_DATABASE}") create_clickhouse_tables(0) + @pytest.mark.async_migrations def test_is_required(self): from posthog.client import sync_execute @@ -61,6 +61,7 @@ class Test0004ReplicatedSchema(AsyncMigrationBaseTest, ClickhouseTestMixin): sync_execute(DISTRIBUTED_EVENTS_TABLE_SQL()) self.assertFalse(migration.is_required()) + @pytest.mark.async_migrations def test_migration(self): # :TRICKY: Relies on tables being migrated as unreplicated before. @@ -83,6 +84,7 @@ class Test0004ReplicatedSchema(AsyncMigrationBaseTest, ClickhouseTestMixin): ) self.assertEqual(self.get_event_table_row_count(), 2) + @pytest.mark.async_migrations def test_rollback(self): # :TRICKY: Relies on tables being migrated as unreplicated before. diff --git a/pytest.ini b/pytest.ini index d0dd7b16ebd..25f9b01f5d9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,4 +8,5 @@ addopts = -p no:warnings --reuse-db markers = ee clickhouse_only - skip_on_multitenancy \ No newline at end of file + skip_on_multitenancy + async_migrations \ No newline at end of file