From 4aa9c4ca65f65eb0b2124ca9a1912b4a3391e62a Mon Sep 17 00:00:00 2001 From: Tim Glaser Date: Sun, 26 Jan 2020 21:54:19 -0800 Subject: [PATCH] Action filtering of events --- README.md | 41 ++++++++++ frontend/editor/index.js | 3 +- posthog/api/action.py | 25 +++++- posthog/api/capture.py | 2 +- posthog/api/event.py | 6 +- posthog/api/test/test_action.py | 2 +- posthog/api/test/test_event.py | 10 +-- posthog/migrations/0009_auto_20200127_0018.py | 18 ++++ posthog/models.py | 82 ++++++++++++++++++- posthog/test/__init__.py | 0 posthog/test/test_event_model.py | 63 ++++++++++++++ 11 files changed, 238 insertions(+), 14 deletions(-) create mode 100644 README.md create mode 100644 posthog/migrations/0009_auto_20200127_0018.py create mode 100644 posthog/test/__init__.py create mode 100644 posthog/test/test_event_model.py diff --git a/README.md b/README.md new file mode 100644 index 00000000000..9170a80cc75 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# Posthog + +## Running locally +1) Make sure you have python 3 installed `python3 --version` +2) Make sure you have postgres installed `brew install postgres` +3) Start postgres, run `brew services start postgresql` +4) Create Database `createdb posthog` +5) Navigate into the correct folder `cd posthog` +6) Run `python3 -m venv env` (creates virtual environment in current direction called 'env') +7) Run `source env/bin/activate` (activates virtual environment) +8) Run `pip install -r requirements.txt` +9) Run migrations `python manage.py migrate` +10) Run `python manage.py createsuperuser` +11) Create a username, email and password +12) Run `python manage.py runserver` + +## Running tests +`bin/tests` + +## Running frontend + +`bin/start-frontend` + +## Pulling production database locally + +`bin/pull_production_db` + +## Create a new branch +If you are working on some changes, please create a new branch, submit it to github ask for approval and when it gets approved it should automatically ship to Heroku + +* Before writing anything run `git pull origin master` +* Then create your branch `git checkout -b %your_branch_name%` call your branch something that represents what you're planning to do +* When you're finished add your changes `git add .` +* And commit with a message `git commit -m "%your feature description%" ` +* When pushing to github make sure you push your branch name and not master!! + +## Deployment to Heroku + +* `git push origin %branch_name%` (sends it to Github) - DO NOT use `git push heroku master` +* Be very careful running migrations by testing if they work locally first (ie run makemigrations, migrate, runserver locally when you've made database changes) +* James or Tim will approve your change, and will deploy it to master diff --git a/frontend/editor/index.js b/frontend/editor/index.js index 922fe8fe439..dcfca0af149 100644 --- a/frontend/editor/index.js +++ b/frontend/editor/index.js @@ -139,6 +139,7 @@ class SelectElement extends Component { + {this.state.step.id}:{this.state.step.isNew}

{this.state.step.href && { - this.state.action.steps = this.state.action.steps.map((s) => (s.id == step.id || (step.isNew && s.isNew == step.isNew)) ? {...step, ...newStep} : s); + this.state.action.steps = this.state.action.steps.map((s) => ((step.id && s.id == step.id) || (step.isNew && s.isNew == step.isNew)) ? {...step, ...newStep} : s); this.setState({action: this.state.action}); }} /> )} diff --git a/posthog/api/action.py b/posthog/api/action.py index 65954c820ef..7549c9c1246 100644 --- a/posthog/api/action.py +++ b/posthog/api/action.py @@ -1,7 +1,9 @@ -from posthog.models import Event, Team, Action, ActionStep +from posthog.models import Event, Team, Action, ActionStep, Element from rest_framework import request, serializers, viewsets # type: ignore from rest_framework.response import Response from rest_framework.decorators import action # type: ignore +from django.db.models import Q, F +from django.forms.models import model_to_dict from typing import Any @@ -38,6 +40,8 @@ class ActionViewSet(viewsets.ModelViewSet): return Response(data={'detail': 'event already exists'}, status=400) for step in steps: + if step.get('isNew'): + step.pop('isNew') ActionStep.objects.create( action=action, **step @@ -66,4 +70,21 @@ class ActionViewSet(viewsets.ModelViewSet): action=action, **step ) - return Response(ActionSerializer(action).data) \ No newline at end of file + return Response(ActionSerializer(action).data) + + + + + def list(self, request: request.Request, *args: Any, **kwargs: Any) -> Response: + actions = self.get_queryset() + actions_list = [] + for action in actions: + count = Event.objects.filter_by_action(action) + actions_list.append({ + 'id': action.pk, + 'name': action.name, + 'count': count + }) + return Response({'results': actions_list}) + + \ No newline at end of file diff --git a/posthog/api/capture.py b/posthog/api/capture.py index eea17ce8ae4..22d9f0577e5 100644 --- a/posthog/api/capture.py +++ b/posthog/api/capture.py @@ -50,7 +50,7 @@ def get_event(request): if elements: Element.objects.bulk_create([ Element( - el_text=el.get('$el_text'), + text=el.get('$el_text'), tag_name=el['tag_name'], href=el.get('attr__href'), attr_id=el.get('attr__id'), diff --git a/posthog/api/event.py b/posthog/api/event.py index 89d5a2db7f0..9242e5c2ed2 100644 --- a/posthog/api/event.py +++ b/posthog/api/event.py @@ -8,7 +8,7 @@ from typing import Any class ElementSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Element - fields = ['el_text', 'tag_name', 'href', 'attr_id', 'nth_child', 'nth_of_type', 'attributes', 'order'] + fields = ['text', 'tag_name', 'href', 'attr_id', 'nth_child', 'nth_of_type', 'attributes', 'order'] class EventSerializer(serializers.HyperlinkedModelSerializer): person = serializers.SerializerMethodField() @@ -74,13 +74,13 @@ class EventViewSet(viewsets.ModelViewSet): def elements(self, request) -> response.Response: elements = Element.objects.filter(team=request.user.team_set.get())\ .filter(tag_name__in=Element.USEFUL_ELEMENTS)\ - .values('tag_name', 'el_text', 'order')\ + .values('tag_name', 'text', 'order')\ .annotate(count=Count('event'))\ .order_by('-count') return response.Response([{ - 'name': '%s with text "%s"' % (el['tag_name'], el['el_text']), + 'name': '%s with text "%s"' % (el['tag_name'], el['text']), 'count': el['count'], 'common': el } for el in elements]) \ No newline at end of file diff --git a/posthog/api/test/test_action.py b/posthog/api/test/test_action.py index 6ec88cd3a65..ee9e590eebf 100644 --- a/posthog/api/test/test_action.py +++ b/posthog/api/test/test_action.py @@ -1,5 +1,5 @@ from .base import BaseTest -from posthog.models import Action, ActionStep +from posthog.models import Action, ActionStep, Event, Element class TestAction(BaseTest): TESTS_API = True diff --git a/posthog/api/test/test_event.py b/posthog/api/test/test_event.py index 1cef295850e..f104eed9c6d 100644 --- a/posthog/api/test/test_event.py +++ b/posthog/api/test/test_event.py @@ -14,7 +14,7 @@ class TestEvents(BaseTest): event1 = Event.objects.create(team=self.team, properties={"distinct_id": "2"}, ip='8.8.8.8') Event.objects.create(team=self.team, properties={"distinct_id": 'some-random-uid'}, ip='8.8.8.8') Event.objects.create(team=self.team, properties={"distinct_id": 'some-other-one'}, ip='8.8.8.8') - Element.objects.create(tag_name='button', el_text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team) + Element.objects.create(tag_name='button', text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team) response = self.client.get('/api/event/?distinct_id=2').json() @@ -40,10 +40,10 @@ class TestEvents(BaseTest): event2 = Event.objects.create(team=self.team, ip='8.8.8.8') event3 = Event.objects.create(team=self.team, ip='8.8.8.8') event4 = Event.objects.create(team=self.team, ip='8.8.8.8') - Element.objects.create(tag_name='button', el_text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team) - Element.objects.create(tag_name='button', el_text='something', nth_child=0, nth_of_type=0, event=event2, order=0, team=self.team) - Element.objects.create(tag_name='button', el_text='something else', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team) - Element.objects.create(tag_name='input', el_text='', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team) + Element.objects.create(tag_name='button', text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team) + Element.objects.create(tag_name='button', text='something', nth_child=0, nth_of_type=0, event=event2, order=0, team=self.team) + Element.objects.create(tag_name='button', text='something else', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team) + Element.objects.create(tag_name='input', text='', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team) response = self.client.get('/api/event/elements/').json() self.assertEqual(response[0]['name'], 'button with text "something"') diff --git a/posthog/migrations/0009_auto_20200127_0018.py b/posthog/migrations/0009_auto_20200127_0018.py new file mode 100644 index 00000000000..c8e59408d04 --- /dev/null +++ b/posthog/migrations/0009_auto_20200127_0018.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.7 on 2020-01-27 00:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('posthog', '0008_action_actionstep'), + ] + + operations = [ + migrations.RenameField( + model_name='element', + old_name='el_text', + new_name='text', + ), + ] diff --git a/posthog/models.py b/posthog/models.py index af8b699c6a7..471e1ee527e 100644 --- a/posthog/models.py +++ b/posthog/models.py @@ -3,8 +3,10 @@ from django.contrib.postgres.fields import JSONField, ArrayField from django.conf import settings from django.contrib.auth.models import AbstractUser from django.dispatch import receiver +from django.forms.models import model_to_dict import secrets +import re class User(AbstractUser): @@ -23,8 +25,86 @@ def create_team_signup_token(sender, instance, created, **kwargs): instance.api_token = secrets.token_urlsafe(10) instance.save() +class EventManager(models.Manager): + def _handle_nth_child(self, index, tag): + nth_child_regex = r"([a-z]+):nth-child\(([0-9]+)\)" + nth_child = re.match(nth_child_regex, tag) + self.where.append("AND E{}.tag_name = %s".format(index)) + self.params.append(nth_child[1]) + self.where.append("AND E{}.nth_child = {}".format(index, nth_child[2])) + + def _handle_id(self, index, tag): + id_regex = r"\[id=\'(.*)']" + result = re.match(id_regex, tag) + self.where.append("AND E{}.attr_id = %s".format(index)) + self.params.append(result[1]) + + def _filter_selector(self, filters): + selector = filters.pop('selector') + tags = selector.split(' > ') + tags.reverse() + for index, tag in enumerate(tags): + if 'nth-child' in tag: + self._handle_nth_child(index, tag) + elif 'id=' in tag: + self._handle_id(index, tag) + else: + self.where.append("AND E{}.tag_name = %s".format(index)) + self.params.append(tag) + if index > 0: + self.joins.append('INNER JOIN posthog_element E{0} ON (posthog_event.id = E{0}.event_id)'.format(index)) + self.where.append('AND E{0}.order = (( E{1}.order + 1))'.format(index, index-1)) + + def _filters(self, filters): + for key, value in filters.items(): + if key != 'action' and key != 'id' and key != 'selector' and value: + self.where.append('AND E0.{} = %s'.format(key)) + self.params.append(value) + + def _step(self, step): + filters = model_to_dict(step) + self.where.append(' OR (1=1 ') + if filters['selector']: + self._filter_selector(filters) + self._filters(filters) + self.where.append(')') + + def _select(self, count): + if count: + return "SELECT COUNT(posthog_event.id) as id FROM posthog_event " + else: + return """ + SELECT "posthog_event"."id", + "posthog_event"."team_id", + "posthog_event"."event", + "posthog_event"."properties", + "posthog_event"."elements", + "posthog_event"."timestamp", + "posthog_event"."ip" + FROM "posthog_event" """ + + def filter_by_action(self, action, count=False): + query = self._select(count=count) + + self.joins = ['INNER JOIN posthog_element E0 ON (posthog_event.id = E0.event_id)'] + self.where = [] + self.params = [] + + for step in action.steps.all(): + self._step(step) + + query += ' '.join(self.joins) + query += ' WHERE 1=2 ' + query += ' '.join(self.where) + events = Event.objects.raw(query, self.params) + if count: + return events[0].id # bit of a hack to get the total count here + return events + + class Event(models.Model): + objects = EventManager() team: models.ForeignKey = models.ForeignKey(Team, on_delete=models.CASCADE) event: models.CharField = models.CharField(max_length=200, null=True, blank=True) properties: JSONField = JSONField(default=dict) @@ -42,7 +122,7 @@ class Person(models.Model): class Element(models.Model): USEFUL_ELEMENTS = ['a', 'button', 'input', 'select', 'textarea', 'label'] - el_text: models.CharField = models.CharField(max_length=400, null=True, blank=True) + text: models.CharField = models.CharField(max_length=400, null=True, blank=True) tag_name: models.CharField = models.CharField(max_length=400, null=True, blank=True) href: models.CharField = models.CharField(max_length=400, null=True, blank=True) attr_id: models.CharField = models.CharField(max_length=400, null=True, blank=True) diff --git a/posthog/test/__init__.py b/posthog/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/posthog/test/test_event_model.py b/posthog/test/test_event_model.py new file mode 100644 index 00000000000..43776abcda7 --- /dev/null +++ b/posthog/test/test_event_model.py @@ -0,0 +1,63 @@ +from posthog.models import Event, Element, Action, ActionStep +from posthog.api.test.base import BaseTest + +class TestEvent(BaseTest): + def test_filter_with_selectors(self): + user = self._create_user('timg') + event1 = Event.objects.create(team=self.team, ip="8.8.8.8") + Element.objects.create(tag_name='div', event=event1, team=self.team, nth_child=0, nth_of_type=0, order=0) + Element.objects.create(tag_name='a', href='/a-url', event=event1, team=self.team, nth_child=1, nth_of_type=0, order=1) + + event2 = Event.objects.create(team=self.team, ip="8.8.8.8") + Element.objects.create(tag_name='a', event=event2, team=self.team, nth_child=2, nth_of_type=0, order=0, attr_id='someId') + Element.objects.create(tag_name='div', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=1) + # make sure elements don't get double counted if they're part of the same event + Element.objects.create(href='/a-url-2', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=2) + + # test direct decendant ordering + action1 = Action.objects.create(team=self.team) + ActionStep.objects.create(action=action1, tag_name='a', selector='div > a') + + events = Event.objects.filter_by_action(action1) + self.assertEqual(len(events), 1) + self.assertEqual(events[0], event2) + + # test :nth-child() + action2 = Action.objects.create(team=self.team) + ActionStep.objects.create(action=action2, tag_name='a', selector='div > a:nth-child(2)') + + events = Event.objects.filter_by_action(action2) + self.assertEqual(len(events), 1) + self.assertEqual(events[0], event2) + + # test [id='someId'] + action3 = Action.objects.create(team=self.team) + ActionStep.objects.create(action=action3, selector="[id='someId']") + + events = Event.objects.filter_by_action(action3) + self.assertEqual(len(events), 1) + self.assertEqual(events[0], event2) + + def test_with_normal_filters(self): + user = self._create_user('tim') + self.client.force_login(user) + event1 = Event.objects.create(team=self.team, ip="8.8.8.8") + Element.objects.create(tag_name='a', href='/a-url', text='some_text', event=event1, team=self.team, nth_child=0, nth_of_type=0, order=0) + + event2 = Event.objects.create(team=self.team, ip="8.8.8.8") + Element.objects.create(tag_name='a', href='/a-url-2', text='some_other_text', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=0) + # make sure elements don't get double counted if they're part of the same event + Element.objects.create(tag_name='div', text='some_other_text', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=1) + + action1 = Action.objects.create(team=self.team) + ActionStep.objects.create(action=action1, href='/a-url', tag_name='a') + ActionStep.objects.create(action=action1, href='/a-url-2') + + + events = Event.objects.filter_by_action(action1) + self.assertEqual(events[0], event1) + self.assertEqual(len(events), 2) + + # test count + events = Event.objects.filter_by_action(action1, count=True) + self.assertEqual(events, 2) \ No newline at end of file