Action filtering of events

README.md

@@ -0,0 +1,41 @@
+# Posthog
+
+## Running locally
+1) Make sure you have python 3 installed `python3 --version`
+2) Make sure you have postgres installed `brew install postgres`
+3) Start postgres, run `brew services start postgresql`
+4) Create Database `createdb posthog`
+5) Navigate into the correct folder `cd posthog`
+6) Run `python3 -m venv env` (creates virtual environment in current direction called 'env')
+7) Run `source env/bin/activate` (activates virtual environment)
+8) Run `pip install -r requirements.txt`
+9) Run migrations `python manage.py migrate`
+10) Run `python manage.py createsuperuser`
+11) Create a username, email and password
+12) Run `python manage.py runserver`
+
+## Running tests
+`bin/tests`
+
+## Running frontend
+
+`bin/start-frontend`
+
+## Pulling production database locally
+
+`bin/pull_production_db`
+
+## Create a new branch
+If you are working on some changes, please create a new branch, submit it to github ask for approval and when it gets approved it should automatically ship to Heroku
+
+* Before writing anything run `git pull origin master`
+* Then create your branch `git checkout -b %your_branch_name%` call your branch something that represents what you're planning to do
+* When you're finished add your changes `git add .`
+* And commit with a message `git commit -m "%your feature description%" `
+* When pushing to github make sure you push your branch name and not master!!
+
+## Deployment to Heroku
+
+* `git push origin %branch_name%` (sends it to Github) - DO NOT use `git push heroku master`
+* Be very careful running migrations by testing if they work locally first (ie run makemigrations, migrate, runserver locally when you've made database changes)
+* James or Tim will approve your change, and will deploy it to master

frontend/editor/index.js

@@ -139,6 +139,7 @@ class SelectElement extends Component {
             <button type="button" className='btn btn-sm btn-light' onClick={() => this.start()}>
                 inspect element
             </button>
+            {this.state.step.id}:{this.state.step.isNew}
             <div style={{margin: '0 -12px'}}>
                 <br />
                 {this.state.step.href && <this.Option
@@ -226,7 +227,7 @@ class App extends Component {
                         this.setState({action: this.state.action});
                     }}
                     onChange={(newStep) => {
-                        this.state.action.steps = this.state.action.steps.map((s) => (s.id == step.id || (step.isNew && s.isNew == step.isNew)) ? {...step, ...newStep} : s);
+                        this.state.action.steps = this.state.action.steps.map((s) => ((step.id && s.id == step.id) || (step.isNew && s.isNew == step.isNew)) ? {...step, ...newStep} : s);
                         this.setState({action: this.state.action});
                     }} />
                 )}

posthog/api/action.py

@@ -1,7 +1,9 @@
-from posthog.models import Event, Team, Action, ActionStep
+from posthog.models import Event, Team, Action, ActionStep, Element
 from rest_framework import request, serializers, viewsets # type: ignore
 from rest_framework.response import Response
 from rest_framework.decorators import action # type: ignore
+from django.db.models import Q, F
+from django.forms.models import model_to_dict
 from typing import Any
 
 
@@ -38,6 +40,8 @@ class ActionViewSet(viewsets.ModelViewSet):
             return Response(data={'detail': 'event already exists'}, status=400)
 
         for step in steps:
+            if step.get('isNew'):
+                step.pop('isNew')
             ActionStep.objects.create(
                 action=action,
                 **step
@@ -66,4 +70,21 @@ class ActionViewSet(viewsets.ModelViewSet):
                     action=action,
                     **step
                 )
-        return Response(ActionSerializer(action).data)
+        return Response(ActionSerializer(action).data)
+
+
+
+
+    def list(self, request: request.Request, *args: Any, **kwargs: Any) -> Response:
+        actions = self.get_queryset()
+        actions_list = []
+        for action in actions:
+            count = Event.objects.filter_by_action(action)
+            actions_list.append({
+                'id': action.pk,
+                'name': action.name,
+                'count': count
+            })
+        return Response({'results': actions_list})
+        
+            

posthog/api/capture.py

@@ -50,7 +50,7 @@ def get_event(request):
     if elements: 
         Element.objects.bulk_create([
             Element(
-                el_text=el.get('$el_text'),
+                text=el.get('$el_text'),
                 tag_name=el['tag_name'],
                 href=el.get('attr__href'),
                 attr_id=el.get('attr__id'),

posthog/api/event.py

@@ -8,7 +8,7 @@ from typing import Any
 class ElementSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = Element
-        fields = ['el_text', 'tag_name', 'href', 'attr_id', 'nth_child', 'nth_of_type', 'attributes', 'order']
+        fields = ['text', 'tag_name', 'href', 'attr_id', 'nth_child', 'nth_of_type', 'attributes', 'order']
 
 class EventSerializer(serializers.HyperlinkedModelSerializer):
     person = serializers.SerializerMethodField()
@@ -74,13 +74,13 @@ class EventViewSet(viewsets.ModelViewSet):
     def elements(self, request) -> response.Response:
         elements = Element.objects.filter(team=request.user.team_set.get())\
             .filter(tag_name__in=Element.USEFUL_ELEMENTS)\
-            .values('tag_name', 'el_text', 'order')\
+            .values('tag_name', 'text', 'order')\
             .annotate(count=Count('event'))\
             .order_by('-count')
 
         
         return response.Response([{
-            'name': '%s with text "%s"' % (el['tag_name'], el['el_text']),
+            'name': '%s with text "%s"' % (el['tag_name'], el['text']),
             'count': el['count'],
             'common': el
         } for el in elements])

posthog/api/test/test_action.py

@@ -1,5 +1,5 @@
 from .base import BaseTest
-from posthog.models import Action, ActionStep
+from posthog.models import Action, ActionStep, Event, Element
 
 class TestAction(BaseTest):
     TESTS_API = True

posthog/api/test/test_event.py

@@ -14,7 +14,7 @@ class TestEvents(BaseTest):
         event1 = Event.objects.create(team=self.team, properties={"distinct_id": "2"}, ip='8.8.8.8')
         Event.objects.create(team=self.team, properties={"distinct_id": 'some-random-uid'}, ip='8.8.8.8')
         Event.objects.create(team=self.team, properties={"distinct_id": 'some-other-one'}, ip='8.8.8.8')
-        Element.objects.create(tag_name='button', el_text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team)
+        Element.objects.create(tag_name='button', text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team)
 
 
         response = self.client.get('/api/event/?distinct_id=2').json()
@@ -40,10 +40,10 @@ class TestEvents(BaseTest):
         event2 = Event.objects.create(team=self.team, ip='8.8.8.8')
         event3 = Event.objects.create(team=self.team, ip='8.8.8.8')
         event4 = Event.objects.create(team=self.team, ip='8.8.8.8')
-        Element.objects.create(tag_name='button', el_text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team)
-        Element.objects.create(tag_name='button', el_text='something', nth_child=0, nth_of_type=0, event=event2, order=0, team=self.team)
-        Element.objects.create(tag_name='button', el_text='something else', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team)
-        Element.objects.create(tag_name='input', el_text='', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team)
+        Element.objects.create(tag_name='button', text='something', nth_child=0, nth_of_type=0, event=event1, order=0, team=self.team)
+        Element.objects.create(tag_name='button', text='something', nth_child=0, nth_of_type=0, event=event2, order=0, team=self.team)
+        Element.objects.create(tag_name='button', text='something else', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team)
+        Element.objects.create(tag_name='input', text='', nth_child=0, nth_of_type=0, event=event3, order=0, team=self.team)
         
         response = self.client.get('/api/event/elements/').json()
         self.assertEqual(response[0]['name'], 'button with text "something"')

posthog/migrations/0009_auto_20200127_0018.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.7 on 2020-01-27 00:18
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('posthog', '0008_action_actionstep'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='element',
+            old_name='el_text',
+            new_name='text',
+        ),
+    ]

posthog/models.py

@@ -3,8 +3,10 @@ from django.contrib.postgres.fields import JSONField, ArrayField
 from django.conf import settings
 from django.contrib.auth.models import AbstractUser
 from django.dispatch import receiver
+from django.forms.models import model_to_dict
 
 import secrets
+import re
 
 
 class User(AbstractUser):
@@ -23,8 +25,86 @@ def create_team_signup_token(sender, instance, created, **kwargs):
             instance.api_token = secrets.token_urlsafe(10)
             instance.save()
 
+class EventManager(models.Manager):
+    def _handle_nth_child(self, index, tag):
+        nth_child_regex =  r"([a-z]+):nth-child\(([0-9]+)\)"
+        nth_child = re.match(nth_child_regex, tag)
+        self.where.append("AND E{}.tag_name = %s".format(index))
+        self.params.append(nth_child[1])
+        self.where.append("AND E{}.nth_child = {}".format(index, nth_child[2]))
+
+    def _handle_id(self, index, tag):
+        id_regex =  r"\[id=\'(.*)']"
+        result = re.match(id_regex, tag)
+        self.where.append("AND E{}.attr_id = %s".format(index))
+        self.params.append(result[1])
+
+    def _filter_selector(self, filters):
+        selector = filters.pop('selector')
+        tags = selector.split(' > ')
+        tags.reverse()
+        for index, tag in enumerate(tags):
+            if 'nth-child' in tag:
+                self._handle_nth_child(index, tag)
+            elif 'id=' in tag:
+                self._handle_id(index, tag)
+            else:
+                self.where.append("AND E{}.tag_name = %s".format(index))
+                self.params.append(tag)
+            if index > 0:
+                self.joins.append('INNER JOIN posthog_element E{0} ON (posthog_event.id = E{0}.event_id)'.format(index))
+                self.where.append('AND E{0}.order = (( E{1}.order + 1))'.format(index, index-1))
+
+    def _filters(self, filters):
+        for key, value in filters.items():
+            if key != 'action' and key != 'id' and key != 'selector' and value:
+                self.where.append('AND E0.{} = %s'.format(key))
+                self.params.append(value)
+
+    def _step(self, step):
+        filters = model_to_dict(step)
+        self.where.append(' OR (1=1 ')
+        if filters['selector']:
+            self._filter_selector(filters)
+        self._filters(filters)
+        self.where.append(')')
+
+    def _select(self, count):
+        if count:
+            return "SELECT COUNT(posthog_event.id) as id FROM posthog_event "
+        else:
+            return """
+            SELECT "posthog_event"."id", 
+                "posthog_event"."team_id", 
+                "posthog_event"."event", 
+                "posthog_event"."properties", 
+                "posthog_event"."elements", 
+                "posthog_event"."timestamp", 
+                "posthog_event"."ip" 
+            FROM   "posthog_event" """
+
+    def filter_by_action(self, action, count=False):
+        query = self._select(count=count)
+        
+        self.joins = ['INNER JOIN posthog_element E0 ON (posthog_event.id = E0.event_id)']
+        self.where = []
+        self.params = []
+
+        for step in action.steps.all():
+            self._step(step)
+
+        query += ' '.join(self.joins)
+        query += ' WHERE 1=2 '
+        query += ' '.join(self.where)
+        events = Event.objects.raw(query, self.params)
+        if count:
+            return events[0].id # bit of a hack to get the total count here
+        return events
+
+
 
 class Event(models.Model):
+    objects = EventManager()
     team: models.ForeignKey = models.ForeignKey(Team, on_delete=models.CASCADE)
     event: models.CharField = models.CharField(max_length=200, null=True, blank=True)
     properties: JSONField = JSONField(default=dict)
@@ -42,7 +122,7 @@ class Person(models.Model):
 
 class Element(models.Model):
     USEFUL_ELEMENTS = ['a', 'button', 'input', 'select', 'textarea', 'label']
-    el_text: models.CharField = models.CharField(max_length=400, null=True, blank=True)
+    text: models.CharField = models.CharField(max_length=400, null=True, blank=True)
     tag_name: models.CharField = models.CharField(max_length=400, null=True, blank=True)
     href: models.CharField = models.CharField(max_length=400, null=True, blank=True)
     attr_id: models.CharField = models.CharField(max_length=400, null=True, blank=True)

posthog/test/test_event_model.py

@@ -0,0 +1,63 @@
+from posthog.models import Event, Element, Action, ActionStep
+from posthog.api.test.base import BaseTest
+
+class TestEvent(BaseTest):
+    def test_filter_with_selectors(self):
+        user = self._create_user('timg')
+        event1 = Event.objects.create(team=self.team, ip="8.8.8.8")
+        Element.objects.create(tag_name='div', event=event1, team=self.team, nth_child=0, nth_of_type=0, order=0)
+        Element.objects.create(tag_name='a', href='/a-url', event=event1, team=self.team, nth_child=1, nth_of_type=0, order=1)
+
+        event2 = Event.objects.create(team=self.team, ip="8.8.8.8")
+        Element.objects.create(tag_name='a', event=event2, team=self.team, nth_child=2, nth_of_type=0, order=0, attr_id='someId')
+        Element.objects.create(tag_name='div', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=1)
+        # make sure elements don't get double counted if they're part of the same event
+        Element.objects.create(href='/a-url-2', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=2)
+
+        # test direct decendant ordering
+        action1 = Action.objects.create(team=self.team)
+        ActionStep.objects.create(action=action1, tag_name='a', selector='div > a')
+
+        events = Event.objects.filter_by_action(action1)
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0], event2)
+
+        # test :nth-child()
+        action2 = Action.objects.create(team=self.team)
+        ActionStep.objects.create(action=action2, tag_name='a', selector='div > a:nth-child(2)')
+
+        events = Event.objects.filter_by_action(action2)
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0], event2)
+
+        # test [id='someId'] 
+        action3 = Action.objects.create(team=self.team)
+        ActionStep.objects.create(action=action3, selector="[id='someId']")
+
+        events = Event.objects.filter_by_action(action3)
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0], event2)
+
+    def test_with_normal_filters(self):
+        user = self._create_user('tim')
+        self.client.force_login(user)
+        event1 = Event.objects.create(team=self.team, ip="8.8.8.8")
+        Element.objects.create(tag_name='a', href='/a-url', text='some_text', event=event1, team=self.team, nth_child=0, nth_of_type=0, order=0)
+
+        event2 = Event.objects.create(team=self.team, ip="8.8.8.8")
+        Element.objects.create(tag_name='a', href='/a-url-2', text='some_other_text', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=0)
+        # make sure elements don't get double counted if they're part of the same event
+        Element.objects.create(tag_name='div', text='some_other_text', event=event2, team=self.team, nth_child=0, nth_of_type=0, order=1)
+
+        action1 = Action.objects.create(team=self.team)
+        ActionStep.objects.create(action=action1, href='/a-url', tag_name='a')
+        ActionStep.objects.create(action=action1, href='/a-url-2')
+
+
+        events = Event.objects.filter_by_action(action1)
+        self.assertEqual(events[0], event1)
+        self.assertEqual(len(events), 2)
+
+        # test count
+        events = Event.objects.filter_by_action(action1, count=True)
+        self.assertEqual(events, 2)