mirrors/wagtail
mirrors/wagtail
0
0
Fork 0
mirror of https://github.com/wagtail/wagtail.git synced 2024-12-01 11:41:20 +01:00
Code Issues Releases Wiki Activity

Added support for indexing relations in wagtailsearch

Browse Source
This commit is contained in:
Karl Hobley 2015-07-08 10:03:05 +01:00 committed by Matt Westcott
parent e18f877286
commit ca83c43e3c
6 changed files with 141 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
wagtail/tests/search/migrations/0002_searchtest_tags.py Normal file
View File

@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
import taggit.managers
class Migration(migrations.Migration):
dependencies = [
('taggit', '0001_initial'),
('searchtests', '0001_initial'),
]
operations = [
migrations.AddField(
model_name='searchtest',
name='tags',
field=taggit.managers.TaggableManager(through='taggit.TaggedItem', verbose_name='Tags', to='taggit.Tag', help_text='A comma-separated list of tags.'),
),
]

7
wagtail/tests/search/models.py
View File

@ -1,5 +1,7 @@
from django.db import models
from taggit.managers import TaggableManager
from wagtail.wagtailsearch import index
@ -8,9 +10,14 @@ class SearchTest(models.Model, index.Indexed):
content = models.TextField()
live = models.BooleanField(default=False)
published_date = models.DateField(null=True)
tags = TaggableManager()
search_fields = [
index.SearchField('title', partial_match=True),
index.RelatedFields('tags', [
index.SearchField('name', partial_match=True),
index.FilterField('slug'),
]),
index.SearchField('content'),
index.SearchField('callable_indexed_field'),
index.FilterField('title'),

8
wagtail/wagtailadmin/taggable.py
View File

@ -18,13 +18,11 @@ class TagSearchable(index.Indexed):
search_fields = (
index.SearchField('title', partial_match=True, boost=10),
index.SearchField('get_tags', partial_match=True, boost=10)
index.RelatedFields('tags', [
index.SearchField('name', partial_match=True, boost=10),
]),
)
@property
def get_tags(self):
return ' '.join([tag.name for tag in self.tags.all()])
@classmethod
def get_indexed_objects(cls):
return super(TagSearchable, cls).get_indexed_objects().prefetch_related('tagged_items__tag')

70
wagtail/wagtailsearch/backends/elasticsearch.py
View File

@ -9,10 +9,11 @@ from django.utils.six.moves.urllib.parse import urlparse
from elasticsearch import Elasticsearch, NotFoundError
from elasticsearch.helpers import bulk
from django.db import models
from django.utils.crypto import get_random_string
from wagtail.wagtailsearch.backends.base import BaseSearch, BaseSearchQuery, BaseSearchResults
from wagtail.wagtailsearch.index import SearchField, FilterField, class_is_indexed
from wagtail.wagtailsearch.index import SearchField, FilterField, RelatedFields, class_is_indexed
from wagtail.utils.deprecation import RemovedInWagtail14Warning
@ -95,25 +96,35 @@ class ElasticSearchMapping(object):
return self.model.indexed_get_content_type()
def get_field_mapping(self, field):
mapping = {'type': self.TYPE_MAP.get(field.get_type(self.model), 'string')}
if isinstance(field, RelatedFields):
mapping = {'type': 'nested', 'properties': {}}
if isinstance(field, SearchField):
if field.boost:
mapping['boost'] = field.boost
for sub_field in field.fields:
sub_field_name, sub_field_mapping = self.get_field_mapping(sub_field)
mapping['properties'][sub_field_name] = sub_field_mapping
if field.partial_match:
mapping['index_analyzer'] = 'edgengram_analyzer'
return field.get_index_name(self.model), mapping
else:
mapping = {'type': self.TYPE_MAP.get(field.get_type(self.model), 'string')}
mapping['include_in_all'] = True
elif isinstance(field, FilterField):
mapping['index'] = 'not_analyzed'
mapping['include_in_all'] = False
if isinstance(field, SearchField):
if field.boost:
mapping['boost'] = field.boost
if 'es_extra' in field.kwargs:
for key, value in field.kwargs['es_extra'].items():
mapping[key] = value
if field.partial_match:
mapping['index_analyzer'] = 'edgengram_analyzer'
return field.get_index_name(self.model), mapping
mapping['include_in_all'] = True
elif isinstance(field, FilterField):
mapping['index'] = 'not_analyzed'
mapping['include_in_all'] = False
if 'es_extra' in field.kwargs:
for key, value in field.kwargs['es_extra'].items():
mapping[key] = value
return field.get_index_name(self.model), mapping
def get_mapping(self):
# Make field list
@ -136,6 +147,21 @@ class ElasticSearchMapping(object):
def get_document_id(self, obj):
return obj.indexed_get_toplevel_content_type() + ':' + str(obj.pk)
def _get_nested_document(self, fields, obj):
doc = {}
partials = []
model = type(obj)
for field in fields:
value = field.get_value(obj)
doc[field.get_index_name(model)] = value
# Check if this field should be added into _partials
if isinstance(field, SearchField) and field.partial_match:
partials.append(value)
return doc, partials
def get_document(self, obj):
# Build document
doc = dict(pk=str(obj.pk), content_type=self.model.indexed_get_content_type())
@ -143,6 +169,20 @@ class ElasticSearchMapping(object):
for field in self.model.get_search_fields():
value = field.get_value(obj)
if isinstance(field, RelatedFields):
if isinstance(value, models.Manager):
nested_docs = []
for nested_obj in value.all():
nested_doc, extra_partials = self._get_nested_document(field.fields, nested_obj)
nested_docs.append(nested_doc)
partials.extend(extra_partials)
value = nested_docs
elif isinstance(value, models.Model):
value, extra_partials = self._get_nested_document(field.fields, value)
partials.extend(extra_partials)
doc[field.get_index_name(self.model)] = value
# Check if this field should be added into _partials

19
wagtail/wagtailsearch/index.py
View File

@ -1,4 +1,5 @@
from django.db import models
from django.db.models.fields.related import RelatedField
from django.apps import apps
@ -136,3 +137,21 @@ class SearchField(BaseField):
class FilterField(BaseField):
suffix = '_filter'
class RelatedFields(object):
def __init__(self, field_name, fields):
self.field_name = field_name
self.fields = fields
def get_index_name(self, cls):
return self.field_name
def get_field(self, cls):
return cls._meta.get_field(self.field_name)
def get_value(self, obj):
field = self.get_field(obj.__class__)
if isinstance(field, RelatedField):
return getattr(obj, self.field_name)

40
wagtail/wagtailsearch/tests/test_elasticsearch_backend.py
View File

@ -721,6 +721,7 @@ class TestElasticSearchMapping(TestCase):
# Create ES document
self.obj = models.SearchTest(title="Hello")
self.obj.save()
self.obj.tags.add("a tag")
def test_get_document_type(self):
self.assertEqual(self.es_mapping.get_document_type(), 'searchtests_searchtest')
@ -741,7 +742,14 @@ class TestElasticSearchMapping(TestCase):
'title': {'type': 'string', 'include_in_all': True, 'index_analyzer': 'edgengram_analyzer'},
'title_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
'content': {'type': 'string', 'include_in_all': True},
'callable_indexed_field': {'type': 'string', 'include_in_all': True}
'callable_indexed_field': {'type': 'string', 'include_in_all': True},
'tags': {
'type': 'nested',
'properties': {
'name': {'type': 'string', 'include_in_all': True, 'index_analyzer': 'edgengram_analyzer'},
'slug_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
}
}
}
}
}
@ -755,17 +763,27 @@ class TestElasticSearchMapping(TestCase):
# Get document
document = self.es_mapping.get_document(self.obj)
# Sort partials
if '_partials' in document:
document['_partials'].sort()
# Check
expected_result = {
'pk': str(self.obj.pk),
'content_type': 'searchtests_searchtest',
'_partials': ['Hello'],
'_partials': ['Hello', 'a tag'],
'live_filter': False,
'published_date_filter': None,
'title': 'Hello',
'title_filter': 'Hello',
'callable_indexed_field': 'Callable',
'content': '',
'tags': [
{
'name': 'a tag',
'slug_filter': 'a-tag',
}
],
}
self.assertDictEqual(document, expected_result)
@ -785,6 +803,7 @@ class TestElasticSearchMappingInheritance(TestCase):
# Create ES document
self.obj = models.SearchTestChild(title="Hello", subtitle="World")
self.obj.save()
self.obj.tags.add("a tag")
def test_get_document_type(self):
self.assertEqual(self.es_mapping.get_document_type(), 'searchtests_searchtest_searchtests_searchtestchild')
@ -810,7 +829,14 @@ class TestElasticSearchMappingInheritance(TestCase):
'title': {'type': 'string', 'include_in_all': True, 'index_analyzer': 'edgengram_analyzer'},
'title_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
'content': {'type': 'string', 'include_in_all': True},
'callable_indexed_field': {'type': 'string', 'include_in_all': True}
'callable_indexed_field': {'type': 'string', 'include_in_all': True},
'tags': {
'type': 'nested',
'properties': {
'name': {'type': 'string', 'include_in_all': True, 'index_analyzer': 'edgengram_analyzer'},
'slug_filter': {'index': 'not_analyzed', 'type': 'string', 'include_in_all': False},
}
}
}
}
}
@ -842,13 +868,19 @@ class TestElasticSearchMappingInheritance(TestCase):
# Inherited
'pk': str(self.obj.pk),
'_partials': ['Hello', 'World'],
'_partials': ['Hello', 'World', 'a tag'],
'live_filter': False,
'published_date_filter': None,
'title': 'Hello',
'title_filter': 'Hello',
'callable_indexed_field': 'Callable',
'content': '',
'tags': [
{
'name': 'a tag',
'slug_filter': 'a-tag',
}
],
}
self.assertDictEqual(document, expected_result)