fix(event-search): make search case insensitive for postgres (#5880)

* chore(dev): make sure we reuse the postgres db in tests

This can be overridden with `--create-db`, see https://pytest-django.readthedocs.io/en/latest/database.html#example-work-flow-with-reuse-db-and-create-db

* refactor(test_event_definition): remove requirement for demo data team

The data takes a long time to import and is a little indirect. Removed
and replaced with some simple db creates

* fix(event-search): make search case insensitive for postgres

It already is for clickhouse which is using ILIKE.

One thing I'm not sure about is why we'd not use postgres tsv here
instead of doing our own tokenising etc. Still may be relevant for data
that we wish to keep in postgres.

Closes https://github.com/PostHog/posthog/issues/5812

Co-authored-by: Harry Waye <harry@scalexp.com>
Co-authored-by: eric <eeoneric@gmail.com>

posthog/api/test/test_event_definition.py

@@ -1,45 +1,64 @@
-import random
-from typing import Dict
+import dataclasses
+from datetime import datetime
+from typing import Any, Dict, List, cast
 
+from freezegun.api import freeze_time
 from rest_framework import status
 
-from posthog.demo import create_demo_team
 from posthog.models import EventDefinition, Organization, Team
+from posthog.models.event import Event
+from posthog.models.user import User
 from posthog.tasks.calculate_event_property_usage import calculate_event_property_usage_for_team
 from posthog.test.base import APIBaseTest
 
 
+@freeze_time("2020-01-02")
 class TestEventDefinitionAPI(APIBaseTest):
 
     demo_team: Team = None  # type: ignore
 
-    EXPECTED_EVENT_DEFINITIONS = [
-        {"name": "installed_app", "volume_30_day": 100, "query_usage_30_day": 0},
-        {"name": "rated_app", "volume_30_day": 73, "query_usage_30_day": 0},
-        {"name": "purchase", "volume_30_day": 16, "query_usage_30_day": 0},
-        {"name": "entered_free_trial", "volume_30_day": 0, "query_usage_30_day": 0},
-        {"name": "watched_movie", "volume_30_day": 87, "query_usage_30_day": 0},
-        {"name": "$pageview", "volume_30_day": 327, "query_usage_30_day": 0},
+    EXPECTED_EVENT_DEFINITIONS: List[Dict[str, Any]] = [
+        {"name": "installed_app", "volume_30_day": 1, "query_usage_30_day": 0},
+        {"name": "rated_app", "volume_30_day": 2, "query_usage_30_day": 0},
+        {"name": "purchase", "volume_30_day": 3, "query_usage_30_day": 0},
+        {"name": "entered_free_trial", "volume_30_day": 7, "query_usage_30_day": 0},
+        {"name": "watched_movie", "volume_30_day": 8, "query_usage_30_day": 0},
+        {"name": "$pageview", "volume_30_day": 9, "query_usage_30_day": 0},
     ]
 
     @classmethod
     def setUpTestData(cls):
-        random.seed(900)
-        super().setUpTestData()
-        cls.demo_team = create_demo_team(cls.organization)
+        cls.organization = create_organization(name="test org")
+        cls.demo_team = create_team(organization=cls.organization)
+        cls.user = create_user("user", "pass", cls.organization)
+
+        for event_definition in cls.EXPECTED_EVENT_DEFINITIONS:
+            create_event_definitions(event_definition["name"], team_id=cls.demo_team.pk)
+            for _ in range(event_definition["volume_30_day"]):
+                emit_event(
+                    event=EventData(
+                        event=event_definition["name"],
+                        team_id=cls.demo_team.pk,
+                        distinct_id="abc",
+                        timestamp=datetime(2020, 1, 1),
+                        properties={},
+                    )
+                )
+
+        # To ensure `volume_30_day` and `query_usage_30_day` are returned non
+        # None, we need to call this task to have them calculated.
         calculate_event_property_usage_for_team(cls.demo_team.pk)
-        cls.user.current_team = cls.demo_team
-        cls.user.save()
 
     def test_list_event_definitions(self):
-
         response = self.client.get("/api/projects/@current/event_definitions/")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.json()["count"], len(self.EXPECTED_EVENT_DEFINITIONS))
         self.assertEqual(len(response.json()["results"]), len(self.EXPECTED_EVENT_DEFINITIONS))
 
         for item in self.EXPECTED_EVENT_DEFINITIONS:
-            response_item: Dict = next((_i for _i in response.json()["results"] if _i["name"] == item["name"]), {})
+            response_item: Dict[str, Any] = next(
+                (_i for _i in response.json()["results"] if _i["name"] == item["name"]), {}
+            )
             self.assertEqual(response_item["volume_30_day"], item["volume_30_day"], item)
             self.assertEqual(response_item["query_usage_30_day"], item["query_usage_30_day"], item)
             self.assertEqual(
@@ -98,6 +117,11 @@ class TestEventDefinitionAPI(APIBaseTest):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         self.assertEqual(response.json()["count"], 2)  # rated app, installed app
 
+        # Search should be case insensitive
+        response = self.client.get("/api/projects/@current/event_definitions/?search=App")
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertEqual(response.json()["count"], 2)  # rated app, installed app
+
         # Fuzzy search 1
         response = self.client.get("/api/projects/@current/event_definitions/?search=free tri")
         self.assertEqual(response.status_code, status.HTTP_200_OK)
@@ -118,3 +142,63 @@ class TestEventDefinitionAPI(APIBaseTest):
         self.assertEqual(response.json()["count"], 1)
         for item in response.json()["results"]:
             self.assertIn(item["name"], ["watched_movie"])
+
+
+def create_organization(name: str) -> Organization:
+    return Organization.objects.create(name=name)
+
+
+def create_user(email: str, password: str, organization: Organization):
+    return User.objects.create_and_join(organization, email, password)
+
+
+def create_team(organization: Organization) -> Team:
+    """
+    This is a helper that just creates a team. It currently uses the orm, but we
+    could use either the api, or django admin to create, to get better parity
+    with real world  scenarios.
+
+    Previously these tests were running `posthog.demo.create_demo_team` which
+    also does a lot of creating of other demo data. This is quite complicated
+    and has a couple of downsides:
+
+      1. the tests take 30 seconds just to startup
+      2. it makes it difficult to see what data is being used
+    """
+    return Team.objects.create(
+        organization=organization,
+        name="Test team",
+        ingested_event=True,
+        completed_snippet_onboarding=True,
+        is_demo=True,
+    )
+
+
+@dataclasses.dataclass
+class EventData:
+    """
+    Little utility struct for creating test event data
+    """
+
+    event: str
+    team_id: int
+    distinct_id: str
+    timestamp: datetime
+    properties: Dict[str, Any]
+
+
+def emit_event(event: EventData) -> Event:
+    """
+    Creates an event, given an event dict. Currently just puts this data
+    directly into the db, but could be created via api to get better parity with
+    real world, and could provide the abstraction over if we are using
+    clickhouse or postgres as the primary backend
+    """
+    return cast(Event, Event.objects.create(**dataclasses.asdict(event)))
+
+
+def create_event_definitions(name: str, team_id: int) -> EventDefinition:
+    """
+    Create event definition for a team.
+    """
+    return EventDefinition.objects.create(name=name, team_id=team_id)

posthog/filters.py

@@ -47,7 +47,7 @@ class TermSearchFilterBackend(filters.BaseFilterBackend):
         for term_idx, search_term in enumerate(search_terms):
             search_filter_query = Q()
             for idx, search_field in enumerate(search_fields):
-                search_filter_query = search_filter_query | Q(**{f"{search_field}__contains": search_term})
+                search_filter_query = search_filter_query | Q(**{f"{search_field}__icontains": search_term})
             term_filter = term_filter & search_filter_query
 
         return queryset.filter(term_filter)

pytest.ini

@@ -1,6 +1,7 @@
 [pytest]
 DJANGO_SETTINGS_MODULE = posthog.settings
-addopts = -p no:warnings
+addopts = -p no:warnings --reuse-db
+
 markers =
     ee
     skip_on_multitenancy