From 605c9cf1a6b19b0e19cef2020c50cf02014e32bd Mon Sep 17 00:00:00 2001 From: Matt Westcott Date: Mon, 23 Oct 2023 17:16:34 +0100 Subject: [PATCH] Drop support for Elasticsearch 5 and 6 --- .github/workflows/test.yml | 114 +- docs/contributing/developing.md | 2 +- docs/releases/6.0.md | 4 + docs/topics/search/backends.md | 20 +- runtests.py | 8 - tox.ini | 8 +- .../tests/test_elasticsearch5_backend.py | 1232 -------------- .../tests/test_elasticsearch6_backend.py | 1442 ----------------- wagtail/test/settings.py | 4 - 9 files changed, 9 insertions(+), 2825 deletions(-) delete mode 100644 wagtail/search/tests/test_elasticsearch5_backend.py delete mode 100644 wagtail/search/tests/test_elasticsearch6_backend.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2c28243c3..a43760ee43 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,9 +28,7 @@ concurrency: # - django 4.2, python 3.12, postgres:15, parallel, DISABLE_TIMEZONE=yes # - django stable/5.0.x, python 3.10, postgres (allow failures) # - django main, python 3.10, postgres:latest, parallel (allow failures) -# - elasticsearch 5, django 3.2, python 3.8, sqlite -# - elasticsearch 6, django 3.2, python 3.8, postgres:latest -# - elasticsearch 7, django 4.1, python 3.8, postgres:latest +# - elasticsearch 7, django 3.2, python 3.8, postgres:latest # - opensearch 2, django 4.1, python 3.9, sqlite # - elasticsearch 8, django 4.2, python 3.10, sqlite, USE_EMAIL_USER_MODEL=yes @@ -198,54 +196,6 @@ jobs: name: coverage-data path: .coverage.* - # https://github.com/elastic/elastic-github-actions doesn't work for Elasticsearch 5, - # but https://github.com/getong/elasticsearch-action does - test-sqlite-elasticsearch5: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: '3.8' - django: 'Django>=3.2,<3.3' - steps: - - name: Configure sysctl limits - run: | - sudo swapoff -a - sudo sysctl -w vm.swappiness=1 - sudo sysctl -w fs.file-max=262144 - sudo sysctl -w vm.max_map_count=262144 - - uses: getong/elasticsearch-action@v1.2 - with: - elasticsearch version: 5.6.9 - host port: 9200 - container port: 9200 - host node port: 9300 - node port: 9300 - discovery type: 'single-node' - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e '.[testing]' --config-settings editable_mode=strict - pip install "${{ matrix.django }}" - pip install "elasticsearch>=5,<6" - pip install certifi - - name: Test - run: | - coverage run --parallel-mode --source wagtail runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch5 - env: - DATABASE_ENGINE: django.db.backends.sqlite3 - - name: Upload coverage data - uses: actions/upload-artifact@v3 - with: - name: coverage-data - path: .coverage.* - test-sqlite-elasticsearch8: runs-on: ubuntu-latest strategy: @@ -294,64 +244,6 @@ jobs: name: coverage-data path: .coverage.* - # https://github.com/getong/elasticsearch-action doesn't work for Elasticsearch 6, - # but https://github.com/elastic/elastic-github-actions does - test-postgres-elasticsearch6: - runs-on: ubuntu-latest - strategy: - matrix: - include: - - python: '3.8' - django: 'Django>=3.2,<3.3' - - services: - postgres: - image: postgres:latest - env: - POSTGRES_PASSWORD: postgres - ports: - - 5432:5432 - options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - - steps: - - name: Configure sysctl limits - run: | - sudo swapoff -a - sudo sysctl -w vm.swappiness=1 - sudo sysctl -w fs.file-max=262144 - sudo sysctl -w vm.max_map_count=262144 - - uses: elastic/elastic-github-actions/elasticsearch@master - with: - stack-version: 6.8.13 - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - cache: 'pip' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install "psycopg2>=2.6" - pip install -e '.[testing]' --config-settings editable_mode=strict - pip install "${{ matrix.django }}" - pip install "elasticsearch>=6,<7" - pip install certifi - - name: Test - run: | - coverage run --parallel-mode --source wagtail runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch6 - env: - DATABASE_ENGINE: django.db.backends.postgresql - DATABASE_HOST: localhost - DATABASE_USER: postgres - DATABASE_PASSWORD: postgres - USE_EMAIL_USER_MODEL: ${{ matrix.emailuser }} - - name: Upload coverage data - uses: actions/upload-artifact@v3 - with: - name: coverage-data - path: .coverage.* - test-postgres-elasticsearch7: runs-on: ubuntu-latest continue-on-error: ${{ matrix.experimental }} @@ -359,7 +251,7 @@ jobs: matrix: include: - python: '3.8' - django: 'Django>=4.1,<4.2' + django: 'Django>=3.2,<3.3' experimental: false services: @@ -460,9 +352,7 @@ jobs: - test-sqlite - test-postgres - test-mysql - - test-sqlite-elasticsearch5 - test-sqlite-elasticsearch8 - - test-postgres-elasticsearch6 - test-postgres-elasticsearch7 - test-sqlite-opensearch2 runs-on: ubuntu-latest diff --git a/docs/contributing/developing.md b/docs/contributing/developing.md index cc9719bd81..7a54931aa4 100644 --- a/docs/contributing/developing.md +++ b/docs/contributing/developing.md @@ -159,7 +159,7 @@ It is also possible to set `DATABASE_DRIVER`, which corresponds to the `driver` ### Testing Elasticsearch -You can test Wagtail against Elasticsearch by passing the argument `--elasticsearch5`, `--elasticsearch6`, `--elasticsearch7` or `--elasticsearch8` (corresponding to the version of Elasticsearch you want to test against): +You can test Wagtail against Elasticsearch by passing the argument `--elasticsearch7` or `--elasticsearch8` (corresponding to the version of Elasticsearch you want to test against): ```sh python runtests.py --elasticsearch8 diff --git a/docs/releases/6.0.md b/docs/releases/6.0.md index 3a4dff8204..da4c56e6e1 100644 --- a/docs/releases/6.0.md +++ b/docs/releases/6.0.md @@ -37,6 +37,10 @@ depth: 1 ## Upgrade considerations - changes affecting all projects +### Support for Elasticsearch 5 and 6 dropped + +The Elasticsearch 5 and 6 backends have been removed. If you are using one of these backends, you will need to upgrade to Elasticsearch 7 or 8 before upgrading to Wagtail 6.0. + ## Upgrade considerations - deprecation of old functionality ## Upgrade considerations - changes affecting Wagtail customisations diff --git a/docs/topics/search/backends.md b/docs/topics/search/backends.md index 746204a530..9a3aa5accc 100644 --- a/docs/topics/search/backends.md +++ b/docs/topics/search/backends.md @@ -37,10 +37,6 @@ If you have disabled auto-update, you must run the [](update_index) command on a ## `ATOMIC_REBUILD` -```{warning} -This option may not work on Elasticsearch version 5.4.x, due to [a bug in the handling of aliases](https://github.com/elastic/elasticsearch/issues/24644) - please upgrade to 5.5 or later. -``` - By default (when using the Elasticsearch backend), when the `update_index` command is run, Wagtail deletes the index and rebuilds it from scratch. This causes the search engine to not return results until the rebuild is complete and is also risky as you can't roll back if an error occurs. Setting the `ATOMIC_REBUILD` setting to `True` makes Wagtail rebuild into a separate index while keeping the old index active until the new one is fully built. When the rebuild is finished, the indexes are swapped atomically and the old index is deleted. @@ -62,23 +58,13 @@ This backend is intended to be used for development and also should be good enou ### Elasticsearch Backend -Elasticsearch versions 5, 6, 7 and 8 are supported. Use the appropriate backend for your version: +Elasticsearch versions 7 and 8 are supported. Use the appropriate backend for your version: -- `wagtail.search.backends.elasticsearch5` (Elasticsearch 5.x) -- `wagtail.search.backends.elasticsearch6` (Elasticsearch 6.x) - `wagtail.search.backends.elasticsearch7` (Elasticsearch 7.x) - `wagtail.search.backends.elasticsearch8` (Elasticsearch 8.x) Prerequisites are the [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) service itself and, via pip, the [elasticsearch-py](https://elasticsearch-py.readthedocs.io/) package. The major version of the package must match the installed version of Elasticsearch: -```sh -pip install "elasticsearch>=5.0.0,<6.0.0" # for Elasticsearch 5.x -``` - -```sh -pip install "elasticsearch>=6.4.0,<7.0.0" # for Elasticsearch 6.x -``` - ```sh pip install "elasticsearch>=7.0.0,<8.0.0" # for Elasticsearch 7.x ``` @@ -87,10 +73,6 @@ pip install "elasticsearch>=7.0.0,<8.0.0" # for Elasticsearch 7.x pip install "elasticsearch>=8.0.0,<9.0.0" # for Elasticsearch 8.x ``` -```{warning} -Version 6.3.1 of the Elasticsearch client library is incompatible with Wagtail. Use 6.4.0 or above. -``` - The backend is configured in settings: ```python diff --git a/runtests.py b/runtests.py index 98ff880122..2019f42982 100755 --- a/runtests.py +++ b/runtests.py @@ -19,8 +19,6 @@ def make_parser(): default="imminent", ) parser.add_argument("--postgres", action="store_true") - parser.add_argument("--elasticsearch5", action="store_true") - parser.add_argument("--elasticsearch6", action="store_true") parser.add_argument("--elasticsearch7", action="store_true") parser.add_argument("--elasticsearch8", action="store_true") parser.add_argument("--emailuser", action="store_true") @@ -61,12 +59,6 @@ def runtests(): if args.postgres: os.environ["DATABASE_ENGINE"] = "django.db.backends.postgresql" - if args.elasticsearch5: - os.environ.setdefault("ELASTICSEARCH_URL", "http://localhost:9200") - os.environ.setdefault("ELASTICSEARCH_VERSION", "5") - elif args.elasticsearch6: - os.environ.setdefault("ELASTICSEARCH_URL", "http://localhost:9200") - os.environ.setdefault("ELASTICSEARCH_VERSION", "6") elif args.elasticsearch7: os.environ.setdefault("ELASTICSEARCH_URL", "http://localhost:9200") os.environ.setdefault("ELASTICSEARCH_VERSION", "7") diff --git a/tox.ini b/tox.ini index 1b3a60237c..8edc919568 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,11 @@ skipsdist = True usedevelop = True -envlist = py{38,39,310,311,312}-dj{32,41,42,50stable,main}-{sqlite,postgres,mysql,mssql}-{elasticsearch8,elasticsearch7,elasticsearch6,elasticsearch5,noelasticsearch}-{customuser,emailuser}-{tz,notz}, +envlist = py{38,39,310,311,312}-dj{32,41,42,50stable,main}-{sqlite,postgres,mysql,mssql}-{elasticsearch8,elasticsearch7,noelasticsearch}-{customuser,emailuser}-{tz,notz}, [testenv] install_command = pip install -e ".[testing]" -U {opts} {packages} commands = - elasticsearch5: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch5 - elasticsearch6: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch6 elasticsearch7: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch7 elasticsearch8: coverage run runtests.py wagtail.search wagtail.documents wagtail.images --elasticsearch8 noelasticsearch: coverage run runtests.py {posargs} @@ -32,10 +30,6 @@ deps = postgres: psycopg2>=2.6 mysql: mysqlclient>=1.4,<2 - elasticsearch5: elasticsearch>=5,<6 - elasticsearch5: certifi - elasticsearch6: elasticsearch>=6.4.0,<7 - elasticsearch6: certifi elasticsearch7: elasticsearch>=7,<8 elasticsearch7: certifi elasticsearch8: elasticsearch>=8,<9 diff --git a/wagtail/search/tests/test_elasticsearch5_backend.py b/wagtail/search/tests/test_elasticsearch5_backend.py deleted file mode 100644 index b8cd681ced..0000000000 --- a/wagtail/search/tests/test_elasticsearch5_backend.py +++ /dev/null @@ -1,1232 +0,0 @@ -import datetime -import json -import unittest -from unittest import mock - -from django.db.models import Q -from django.test import TestCase - -from wagtail.search.query import MATCH_ALL, Fuzzy, Phrase -from wagtail.test.search import models - -from .elasticsearch_common_tests import ElasticsearchCommonSearchBackendTests - -try: - from elasticsearch import VERSION as ELASTICSEARCH_VERSION - from elasticsearch.serializer import JSONSerializer - - from wagtail.search.backends.elasticsearch5 import Elasticsearch5SearchBackend -except ImportError: - ELASTICSEARCH_VERSION = (0, 0, 0) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 5, "Elasticsearch 5 required") -class TestElasticsearch5SearchBackend(ElasticsearchCommonSearchBackendTests, TestCase): - backend_path = "wagtail.search.backends.elasticsearch5" - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 5, "Elasticsearch 5 required") -class TestElasticsearch5SearchQuery(TestCase): - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual( - json.dumps(a, sort_keys=True, default=default), - json.dumps(b, sort_keys=True, default=default), - ) - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.query_compiler_class = Elasticsearch5SearchBackend.query_compiler_class - cls.autocomplete_query_compiler_class = ( - Elasticsearch5SearchBackend.autocomplete_query_compiler_class - ) - - def test_simple(self): - # Create a query - query_compiler = self.query_compiler_class(models.Book.objects.all(), "Hello") - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_simple_autocomplete(self): - # Create a query - query_compiler = self.autocomplete_query_compiler_class( - models.Book.objects.all(), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": {"match": {"_partials": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_match_all(self): - # Create a query - query_compiler = self.query_compiler_class(models.Book.objects.all(), MATCH_ALL) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": {"match_all": {}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_and_operator(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), "Hello", operator="and" - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "match": { - "_all": { - "query": "Hello", - "operator": "and", - } - } - }, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_filter(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(title="Test"), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"term": {"title_filter": "Test"}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_and_filter(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter( - title="Test", publication_date=datetime.date(2017, 10, 18) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "must": [ - {"term": {"publication_date_filter": "2017-10-18"}}, - {"term": {"title_filter": "Test"}}, - ] - } - }, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - - # Make sure field filters are sorted (as they can be in any order which may cause false positives) - query = query_compiler.get_query() - field_filters = query["bool"]["filter"][1]["bool"]["must"] - field_filters[:] = sorted( - field_filters, key=lambda f: list(f["term"].keys())[0] - ) - - self.assertDictEqual(query, expected_result) - - def test_or_filter(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter( - Q(title="Test") | Q(publication_date=datetime.date(2017, 10, 18)) - ), - "Hello", - ) - - # Make sure field filters are sorted (as they can be in any order which may cause false positives) - query = query_compiler.get_query() - field_filters = query["bool"]["filter"][1]["bool"]["should"] - field_filters[:] = sorted( - field_filters, key=lambda f: list(f["term"].keys())[0] - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "should": [ - {"term": {"publication_date_filter": "2017-10-18"}}, - {"term": {"title_filter": "Test"}}, - ] - } - }, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query, expected_result) - - def test_negated_filter(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.exclude(publication_date=datetime.date(2017, 10, 18)), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "mustNot": { - "term": {"publication_date_filter": "2017-10-18"} - } - } - }, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_fields(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), "Hello", fields=["title"] - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": {"match": {"title": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_fields_with_and_operator(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), "Hello", fields=["title"], operator="and" - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": {"match": {"title": {"query": "Hello", "operator": "and"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_multiple_fields(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), "Hello", fields=["title", "content"] - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": {"fields": ["title", "content"], "query": "Hello"} - }, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_multiple_fields_with_and_operator(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - "Hello", - fields=["title", "content"], - operator="and", - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": ["title", "content"], - "query": "Hello", - "operator": "and", - } - }, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_exact_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(title__exact="Test"), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"term": {"title_filter": "Test"}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_none_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(title=None), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"bool": {"mustNot": {"exists": {"field": "title_filter"}}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_isnull_true_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(title__isnull=True), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"bool": {"mustNot": {"exists": {"field": "title_filter"}}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_isnull_false_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(title__isnull=False), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"exists": {"field": "title_filter"}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_startswith_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(title__startswith="Test"), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"prefix": {"title_filter": "Test"}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_gt_lookup(self): - # This also tests conversion of python dates to strings - - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter( - publication_date__gt=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"gt": "2014-04-29"}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_lt_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter( - publication_date__lt=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"lt": "2014-04-29"}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_gte_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter( - publication_date__gte=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"gte": "2014-04-29"}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_lte_lookup(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter( - publication_date__lte=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"lte": "2014-04-29"}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_range_lookup(self): - start_date = datetime.datetime(2014, 4, 29) - end_date = datetime.datetime(2014, 8, 19) - - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(publication_date__range=(start_date, end_date)), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "range": { - "publication_date_filter": { - "gte": "2014-04-29", - "lte": "2014-08-19", - } - } - }, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - def test_custom_ordering(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.order_by("publication_date"), - "Hello", - order_by_relevance=False, - ) - - # Check it - expected_result = [{"publication_date_filter": "asc"}] - self.assertDictEqual(query_compiler.get_sort(), expected_result) - - def test_custom_ordering_reversed(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.order_by("-publication_date"), - "Hello", - order_by_relevance=False, - ) - - # Check it - expected_result = [{"publication_date_filter": "desc"}] - self.assertDictEqual(query_compiler.get_sort(), expected_result) - - def test_custom_ordering_multiple(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.order_by("publication_date", "number_of_pages"), - "Hello", - order_by_relevance=False, - ) - - # Check it - expected_result = [ - {"publication_date_filter": "asc"}, - {"number_of_pages_filter": "asc"}, - ] - self.assertDictEqual(query_compiler.get_sort(), expected_result) - - def test_phrase_query(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), Phrase("Hello world") - ) - - # Check it - expected_result = { - "match_phrase": { - "_all": "Hello world", - } - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_phrase_query_multiple_fields(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Phrase("Hello world"), - fields=["title", "content"], - ) - - # Check it - expected_result = { - "multi_match": { - "fields": ["title", "content"], - "query": "Hello world", - "type": "phrase", - } - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_phrase_query_single_field(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), Phrase("Hello world"), fields=["title"] - ) - - # Check it - expected_result = {"match_phrase": {"title": "Hello world"}} - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_fuzzy_query(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Fuzzy("Hello world"), - ) - - # Check it - expected_result = { - "match": {"_all": {"query": "Hello world", "fuzziness": "AUTO"}} - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_fuzzy_query_single_field(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Fuzzy("Hello world"), - fields=["title"], - ) - - # Check it - expected_result = { - "match": {"title": {"query": "Hello world", "fuzziness": "AUTO"}} - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_fuzzy_query_multiple_fields_disallowed(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Fuzzy("Hello world"), - fields=["title", "body"], - ) - - # Check it - with self.assertRaises(NotImplementedError): - query_compiler.get_inner_query() - - def test_year_filter(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(publication_date__year__gt=1900), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"gte": "1901-01-01"}}}, - ], - "must": {"match": {"_all": {"query": "Hello"}}}, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 5, "Elasticsearch 5 required") -class TestElasticsearch5SearchResults(TestCase): - fixtures = ["search"] - - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps) - - def get_results(self): - backend = Elasticsearch5SearchBackend({}) - query_compiler = mock.MagicMock() - query_compiler.queryset = models.Book.objects.all() - query_compiler.get_query.return_value = "QUERY" - query_compiler.get_sort.return_value = None - return backend.results_class(backend, query_compiler) - - def construct_search_response(self, results): - return { - "_shards": {"failed": 0, "successful": 5, "total": 5}, - "hits": { - "hits": [ - { - "_id": "searchtests_book:" + str(result), - "_index": "wagtail", - "_score": 1, - "_type": "searchtests_book", - "fields": { - "pk": [str(result)], - }, - } - for result in results - ], - "max_score": 1, - "total": len(results), - }, - "timed_out": False, - "took": 2, - } - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_basic_search(self, search): - search.return_value = self.construct_search_response([]) - results = self.get_results() - - list(results) # Performs search - - search.assert_any_call( - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - scroll="2m", - size=100, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_get_single_item(self, search): - # Need to return something to prevent index error - search.return_value = self.construct_search_response([1]) - results = self.get_results() - - results[10] # Performs search - - search.assert_any_call( - from_=10, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=1, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_slice_results(self, search): - search.return_value = self.construct_search_response([]) - results = self.get_results()[1:4] - - list(results) # Performs search - - search.assert_any_call( - from_=1, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=3, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_slice_results_multiple_times(self, search): - search.return_value = self.construct_search_response([]) - results = self.get_results()[10:][:10] - - list(results) # Performs search - - search.assert_any_call( - from_=10, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=10, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_slice_results_and_get_item(self, search): - # Need to return something to prevent index error - search.return_value = self.construct_search_response([1]) - results = self.get_results()[10:] - - results[10] # Performs search - - search.assert_any_call( - from_=20, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=1, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_result_returned(self, search): - search.return_value = self.construct_search_response([1]) - results = self.get_results() - - self.assertEqual(results[0], models.Book.objects.get(id=1)) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_len_1(self, search): - search.return_value = self.construct_search_response([1]) - results = self.get_results() - - self.assertEqual(len(results), 1) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_len_2(self, search): - search.return_value = self.construct_search_response([1, 2]) - results = self.get_results() - - self.assertEqual(len(results), 2) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_duplicate_results(self, search): # Duplicates will not be removed - search.return_value = self.construct_search_response([1, 1]) - results = list( - self.get_results() - ) # Must cast to list so we only create one query - - self.assertEqual(len(results), 2) - self.assertEqual(results[0], models.Book.objects.get(id=1)) - self.assertEqual(results[1], models.Book.objects.get(id=1)) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_result_order(self, search): - search.return_value = self.construct_search_response([1, 2, 3]) - results = list( - self.get_results() - ) # Must cast to list so we only create one query - - self.assertEqual(results[0], models.Book.objects.get(id=1)) - self.assertEqual(results[1], models.Book.objects.get(id=2)) - self.assertEqual(results[2], models.Book.objects.get(id=3)) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_result_order_2(self, search): - search.return_value = self.construct_search_response([3, 2, 1]) - results = list( - self.get_results() - ) # Must cast to list so we only create one query - - self.assertEqual(results[0], models.Book.objects.get(id=3)) - self.assertEqual(results[1], models.Book.objects.get(id=2)) - self.assertEqual(results[2], models.Book.objects.get(id=1)) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 5, "Elasticsearch 5 required") -class TestElasticsearch5Mapping(TestCase): - fixtures = ["search"] - - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual( - json.dumps(a, sort_keys=True, default=default), - json.dumps(b, sort_keys=True, default=default), - ) - - def setUp(self): - # Create ES mapping - self.es_mapping = Elasticsearch5SearchBackend.mapping_class(models.Book) - - # Create ES document - self.obj = models.Book.objects.get(id=4) - - def test_get_document_type(self): - self.assertEqual(self.es_mapping.get_document_type(), "searchtests_book") - - def test_get_mapping(self): - # Build mapping - mapping = self.es_mapping.get_mapping() - - # Check - expected_result = { - "searchtests_book": { - "properties": { - "pk": {"type": "keyword", "store": True, "include_in_all": False}, - "content_type": {"type": "keyword", "include_in_all": False}, - "_partials": { - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - "include_in_all": False, - "type": "text", - }, - "title": { - "type": "text", - "boost": 2.0, - "include_in_all": True, - }, - "title_edgengrams": { - "type": "text", - "include_in_all": False, - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - }, - "title_filter": {"type": "keyword", "include_in_all": False}, - "authors": { - "type": "nested", - "properties": { - "name": {"type": "text", "include_in_all": True}, - "name_edgengrams": { - "analyzer": "edgengram_analyzer", - "include_in_all": False, - "search_analyzer": "standard", - "type": "text", - }, - "date_of_birth_filter": { - "type": "date", - "include_in_all": False, - }, - }, - }, - "authors_filter": {"type": "integer", "include_in_all": False}, - "publication_date_filter": { - "type": "date", - "include_in_all": False, - }, - "number_of_pages_filter": { - "type": "integer", - "include_in_all": False, - }, - "tags": { - "type": "nested", - "properties": { - "name": {"type": "text", "include_in_all": True}, - "slug_filter": {"type": "keyword", "include_in_all": False}, - }, - }, - "tags_filter": {"type": "integer", "include_in_all": False}, - } - } - } - - self.assertDictEqual(mapping, expected_result) - - def test_get_document_id(self): - self.assertEqual( - self.es_mapping.get_document_id(self.obj), - "searchtests_book:" + str(self.obj.pk), - ) - - def test_get_document(self): - # Get document - document = self.es_mapping.get_document(self.obj) - - # Sort partials - if "_partials" in document: - document["_partials"].sort() - - # Check - expected_result = { - "pk": "4", - "content_type": ["searchtests.Book"], - "_partials": [ - "J. R. R. Tolkien", - "The Fellowship of the Ring", - ], - "title": "The Fellowship of the Ring", - "title_edgengrams": "The Fellowship of the Ring", - "title_filter": "The Fellowship of the Ring", - "authors": [ - { - "name": "J. R. R. Tolkien", - "name_edgengrams": "J. R. R. Tolkien", - "date_of_birth_filter": datetime.date(1892, 1, 3), - } - ], - "authors_filter": [2], - "publication_date_filter": datetime.date(1954, 7, 29), - "number_of_pages_filter": 423, - "tags": [], - "tags_filter": [], - } - - self.assertDictEqual(document, expected_result) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 5, "Elasticsearch 5 required") -class TestElasticsearch5MappingInheritance(TestCase): - fixtures = ["search"] - - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual( - json.dumps(a, sort_keys=True, default=default), - json.dumps(b, sort_keys=True, default=default), - ) - - def setUp(self): - # Create ES mapping - self.es_mapping = Elasticsearch5SearchBackend.mapping_class(models.Novel) - - self.obj = models.Novel.objects.get(id=4) - - def test_get_document_type(self): - self.assertEqual( - self.es_mapping.get_document_type(), "searchtests_book_searchtests_novel" - ) - - def test_get_mapping(self): - # Build mapping - mapping = self.es_mapping.get_mapping() - - # Check - expected_result = { - "searchtests_book_searchtests_novel": { - "properties": { - # New - "searchtests_novel__setting": { - "type": "text", - "include_in_all": True, - }, - "searchtests_novel__setting_edgengrams": { - "type": "text", - "include_in_all": False, - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - }, - "searchtests_novel__protagonist": { - "type": "nested", - "properties": { - "name": { - "type": "text", - "boost": 0.5, - "include_in_all": True, - }, - "novel_id_filter": { - "type": "integer", - "include_in_all": False, - }, - }, - }, - "searchtests_novel__protagonist_id_filter": { - "type": "integer", - "include_in_all": False, - }, - "searchtests_novel__characters": { - "type": "nested", - "properties": { - "name": { - "type": "text", - "boost": 0.25, - "include_in_all": True, - } - }, - }, - # Inherited - "pk": {"type": "keyword", "store": True, "include_in_all": False}, - "content_type": {"type": "keyword", "include_in_all": False}, - "_partials": { - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - "include_in_all": False, - "type": "text", - }, - "title": { - "type": "text", - "boost": 2.0, - "include_in_all": True, - }, - "title_edgengrams": { - "type": "text", - "include_in_all": False, - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - }, - "title_filter": {"type": "keyword", "include_in_all": False}, - "authors": { - "type": "nested", - "properties": { - "name": {"type": "text", "include_in_all": True}, - "name_edgengrams": { - "analyzer": "edgengram_analyzer", - "include_in_all": False, - "search_analyzer": "standard", - "type": "text", - }, - "date_of_birth_filter": { - "type": "date", - "include_in_all": False, - }, - }, - }, - "authors_filter": {"type": "integer", "include_in_all": False}, - "publication_date_filter": { - "type": "date", - "include_in_all": False, - }, - "number_of_pages_filter": { - "type": "integer", - "include_in_all": False, - }, - "tags": { - "type": "nested", - "properties": { - "name": {"type": "text", "include_in_all": True}, - "slug_filter": {"type": "keyword", "include_in_all": False}, - }, - }, - "tags_filter": {"type": "integer", "include_in_all": False}, - } - } - } - - self.assertDictEqual(mapping, expected_result) - - def test_get_document_id(self): - # This must be tests_searchtest instead of 'tests_searchtest_tests_searchtestchild' - # as it uses the contents base content type name. - # This prevents the same object being accidentally indexed twice. - self.assertEqual( - self.es_mapping.get_document_id(self.obj), - "searchtests_book:" + str(self.obj.pk), - ) - - def test_get_document(self): - # Build document - document = self.es_mapping.get_document(self.obj) - - # Sort partials - if "_partials" in document: - document["_partials"].sort() - - # Sort characters - if "searchtests_novel__characters" in document: - document["searchtests_novel__characters"].sort(key=lambda c: c["name"]) - - # Check - expected_result = { - # New - "searchtests_novel__setting": "Middle Earth", - "searchtests_novel__setting_edgengrams": "Middle Earth", - "searchtests_novel__protagonist": { - "name": "Frodo Baggins", - "novel_id_filter": 4, - }, - "searchtests_novel__protagonist_id_filter": 8, - "searchtests_novel__characters": [ - {"name": "Bilbo Baggins"}, - {"name": "Frodo Baggins"}, - {"name": "Gandalf"}, - ], - # Changed - "content_type": ["searchtests.Novel", "searchtests.Book"], - "_partials": [ - "J. R. R. Tolkien", - "Middle Earth", - "The Fellowship of the Ring", - ], - # Inherited - "pk": "4", - "title": "The Fellowship of the Ring", - "title_edgengrams": "The Fellowship of the Ring", - "title_filter": "The Fellowship of the Ring", - "authors": [ - { - "name": "J. R. R. Tolkien", - "name_edgengrams": "J. R. R. Tolkien", - "date_of_birth_filter": datetime.date(1892, 1, 3), - } - ], - "authors_filter": [2], - "publication_date_filter": datetime.date(1954, 7, 29), - "number_of_pages_filter": 423, - "tags": [], - "tags_filter": [], - } - - self.assertDictEqual(document, expected_result) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 5, "Elasticsearch 5 required") -@mock.patch("wagtail.search.backends.elasticsearch5.Elasticsearch") -class TestBackendConfiguration(TestCase): - def test_default_settings(self, Elasticsearch): - Elasticsearch5SearchBackend(params={}) - - Elasticsearch.assert_called_with( - hosts=[ - { - "host": "localhost", - "port": 9200, - "url_prefix": "", - "use_ssl": False, - "verify_certs": False, - "http_auth": None, - } - ], - timeout=10, - ) - - def test_hosts(self, Elasticsearch): - Elasticsearch5SearchBackend( - params={ - "HOSTS": [ - { - "host": "127.0.0.1", - "port": 9300, - "use_ssl": True, - "verify_certs": True, - } - ] - } - ) - - Elasticsearch.assert_called_with( - hosts=[ - { - "host": "127.0.0.1", - "port": 9300, - "use_ssl": True, - "verify_certs": True, - } - ], - timeout=10, - ) - - def test_urls(self, Elasticsearch): - # This test backwards compatibility with old URLS setting - Elasticsearch5SearchBackend( - params={ - "URLS": [ - "http://localhost:12345", - "https://127.0.0.1:54321", - "http://username:password@elasticsearch.mysite.com", - "https://elasticsearch.mysite.com/hello", - ], - } - ) - - Elasticsearch.assert_called_with( - hosts=[ - { - "host": "localhost", - "port": 12345, - "url_prefix": "", - "use_ssl": False, - "verify_certs": False, - "http_auth": None, - }, - { - "host": "127.0.0.1", - "port": 54321, - "url_prefix": "", - "use_ssl": True, - "verify_certs": True, - "http_auth": None, - }, - { - "host": "elasticsearch.mysite.com", - "port": 80, - "url_prefix": "", - "use_ssl": False, - "verify_certs": False, - "http_auth": ("username", "password"), - }, - { - "host": "elasticsearch.mysite.com", - "port": 443, - "url_prefix": "/hello", - "use_ssl": True, - "verify_certs": True, - "http_auth": None, - }, - ], - timeout=10, - ) diff --git a/wagtail/search/tests/test_elasticsearch6_backend.py b/wagtail/search/tests/test_elasticsearch6_backend.py deleted file mode 100644 index 4b84b8a821..0000000000 --- a/wagtail/search/tests/test_elasticsearch6_backend.py +++ /dev/null @@ -1,1442 +0,0 @@ -import datetime -import json -import unittest -from unittest import mock - -from django.db.models import Q -from django.test import TestCase - -from wagtail.search.query import MATCH_ALL, Fuzzy, Phrase -from wagtail.test.search import models - -from .elasticsearch_common_tests import ElasticsearchCommonSearchBackendTests - -try: - from elasticsearch import VERSION as ELASTICSEARCH_VERSION - from elasticsearch.serializer import JSONSerializer - - from wagtail.search.backends.elasticsearch6 import Elasticsearch6SearchBackend -except ImportError: - ELASTICSEARCH_VERSION = (0, 0, 0) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 6, "Elasticsearch 6 required") -class TestElasticsearch6SearchBackend(ElasticsearchCommonSearchBackendTests, TestCase): - backend_path = "wagtail.search.backends.elasticsearch6" - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 6, "Elasticsearch 6 required") -class TestElasticsearch6SearchQuery(TestCase): - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual( - json.dumps(a, sort_keys=True, default=default), - json.dumps(b, sort_keys=True, default=default), - ) - - @classmethod - def setUpClass(cls): - super().setUpClass() - cls.query_compiler_class = Elasticsearch6SearchBackend.query_compiler_class - cls.autocomplete_query_compiler_class = ( - Elasticsearch6SearchBackend.autocomplete_query_compiler_class - ) - - def test_simple(self): - # Create a query - query = self.query_compiler_class(models.Book.objects.all(), "Hello") - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_simple_autocomplete(self): - # Create a query - query = self.autocomplete_query_compiler_class( - models.Book.objects.all(), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "match": { - "_edgengrams": { - "query": "Hello", - } - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_none_query_string(self): - # Create a query - query = self.query_compiler_class(models.Book.objects.all(), MATCH_ALL) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": {"match_all": {}}, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_and_operator(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.all(), "Hello", operator="and" - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - "operator": "and", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_filter(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(title="Test"), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"term": {"title_filter": "Test"}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_and_filter(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter( - title="Test", publication_date=datetime.date(2017, 10, 18) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "must": [ - {"term": {"publication_date_filter": "2017-10-18"}}, - {"term": {"title_filter": "Test"}}, - ] - } - }, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - - # Make sure field filters are sorted (as they can be in any order which may cause false positives) - query = query.get_query() - field_filters = query["bool"]["filter"][1]["bool"]["must"] - field_filters[:] = sorted( - field_filters, key=lambda f: list(f["term"].keys())[0] - ) - - self.assertDictEqual(query, expected_result) - - def test_or_filter(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter( - Q(title="Test") | Q(publication_date=datetime.date(2017, 10, 18)) - ), - "Hello", - ) - - # Make sure field filters are sorted (as they can be in any order which may cause false positives) - query = query.get_query() - field_filters = query["bool"]["filter"][1]["bool"]["should"] - field_filters[:] = sorted( - field_filters, key=lambda f: list(f["term"].keys())[0] - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "should": [ - {"term": {"publication_date_filter": "2017-10-18"}}, - {"term": {"title_filter": "Test"}}, - ] - } - }, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query, expected_result) - - def test_negated_filter(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.exclude(publication_date=datetime.date(2017, 10, 18)), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "mustNot": { - "term": {"publication_date_filter": "2017-10-18"} - } - } - }, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_fields(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.all(), "Hello", fields=["title"] - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": [ - "title", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_fields_with_and_operator(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.all(), "Hello", fields=["title"], operator="and" - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": [ - "title", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - "operator": "and", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_multiple_fields(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.all(), "Hello", fields=["title", "content"] - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": [ - "title", - "content", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_multiple_fields_with_and_operator(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.all(), - "Hello", - fields=["title", "content"], - operator="and", - ) - - # Check it - expected_result = { - "bool": { - "filter": {"match": {"content_type": "searchtests.Book"}}, - "must": { - "multi_match": { - "fields": [ - "title", - "content", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - "operator": "and", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_exact_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(title__exact="Test"), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"term": {"title_filter": "Test"}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_none_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(title=None), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"bool": {"mustNot": {"exists": {"field": "title_filter"}}}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_isnull_true_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(title__isnull=True), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"bool": {"mustNot": {"exists": {"field": "title_filter"}}}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_isnull_false_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(title__isnull=False), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"exists": {"field": "title_filter"}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_startswith_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(title__startswith="Test"), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"prefix": {"title_filter": "Test"}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_gt_lookup(self): - # This also tests conversion of python dates to strings - - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter( - publication_date__gt=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"gt": "2014-04-29"}}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_lt_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter( - publication_date__lt=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"lt": "2014-04-29"}}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_gte_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter( - publication_date__gte=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"gte": "2014-04-29"}}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_lte_lookup(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter( - publication_date__lte=datetime.datetime(2014, 4, 29) - ), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - {"range": {"publication_date_filter": {"lte": "2014-04-29"}}}, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_range_lookup(self): - start_date = datetime.datetime(2014, 4, 29) - end_date = datetime.datetime(2014, 8, 19) - - # Create a query - query = self.query_compiler_class( - models.Book.objects.filter(publication_date__range=(start_date, end_date)), - "Hello", - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "range": { - "publication_date_filter": { - "gte": "2014-04-29", - "lte": "2014-08-19", - } - } - }, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query.get_query(), expected_result) - - def test_custom_ordering(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.order_by("publication_date"), - "Hello", - order_by_relevance=False, - ) - - # Check it - expected_result = [{"publication_date_filter": "asc"}] - self.assertDictEqual(query.get_sort(), expected_result) - - def test_custom_ordering_reversed(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.order_by("-publication_date"), - "Hello", - order_by_relevance=False, - ) - - # Check it - expected_result = [{"publication_date_filter": "desc"}] - self.assertDictEqual(query.get_sort(), expected_result) - - def test_custom_ordering_multiple(self): - # Create a query - query = self.query_compiler_class( - models.Book.objects.order_by("publication_date", "number_of_pages"), - "Hello", - order_by_relevance=False, - ) - - # Check it - expected_result = [ - {"publication_date_filter": "asc"}, - {"number_of_pages_filter": "asc"}, - ] - self.assertDictEqual(query.get_sort(), expected_result) - - def test_phrase_query(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), Phrase("Hello world") - ) - - # Check it - expected_result = { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello world", - "type": "phrase", - }, - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_phrase_query_multiple_fields(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Phrase("Hello world"), - fields=["title", "content"], - ) - - # Check it - expected_result = { - "multi_match": { - "fields": [ - "title", - "content", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello world", - "type": "phrase", - }, - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_phrase_query_single_field(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), Phrase("Hello world"), fields=["title"] - ) - - # Check it - expected_result = { - "multi_match": { - "fields": [ - "title", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello world", - "type": "phrase", - }, - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_fuzzy_query(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Fuzzy("Hello world"), - ) - - # Check it - expected_result = { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello world", - "fuzziness": "AUTO", - }, - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_fuzzy_query_single_field(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Fuzzy("Hello world"), - fields=["title"], - ) - - # Check it - expected_result = { - "multi_match": { - "fields": [ - "title", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello world", - "fuzziness": "AUTO", - }, - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_fuzzy_query_multiple_fields(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.all(), - Fuzzy("Hello world"), - fields=["title", "body"], - ) - - expected_result = { - "multi_match": { - "fields": [ - "title", - "body", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello world", - "fuzziness": "AUTO", - }, - } - self.assertDictEqual(query_compiler.get_inner_query(), expected_result) - - def test_year_filter(self): - # Create a query - query_compiler = self.query_compiler_class( - models.Book.objects.filter(publication_date__year=1900), "Hello" - ) - - # Check it - expected_result = { - "bool": { - "filter": [ - {"match": {"content_type": "searchtests.Book"}}, - { - "bool": { - "must": [ - { - "range": { - "publication_date_filter": {"gte": "1900-01-01"} - } - }, - { - "range": { - "publication_date_filter": {"lt": "1901-01-01"} - } - }, - ] - } - }, - ], - "must": { - "multi_match": { - "fields": [ - "_all_text", - "_all_text_boost_2_0^2.0", - "_all_text_boost_10_0^10.0", - ], - "query": "Hello", - }, - }, - } - } - self.assertDictEqual(query_compiler.get_query(), expected_result) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 6, "Elasticsearch 6 required") -class TestElasticsearch6SearchResults(TestCase): - fixtures = ["search"] - - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual(json.dumps(a, sort_keys=True, default=default), json.dumps) - - def get_results(self): - backend = Elasticsearch6SearchBackend({}) - query = mock.MagicMock() - query.queryset = models.Book.objects.all() - query.get_query.return_value = "QUERY" - query.get_sort.return_value = None - return backend.results_class(backend, query) - - def construct_search_response(self, results): - return { - "_shards": {"failed": 0, "successful": 5, "total": 5}, - "hits": { - "hits": [ - { - "_id": "searchtests_book:" + str(result), - "_index": "wagtail", - "_score": 1, - "_type": "searchtests_book", - "fields": { - "pk": [str(result)], - }, - } - for result in results - ], - "max_score": 1, - "total": len(results), - }, - "timed_out": False, - "took": 2, - } - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_basic_search(self, search): - search.return_value = self.construct_search_response([]) - results = self.get_results() - - list(results) # Performs search - - search.assert_any_call( - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - scroll="2m", - size=100, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_get_single_item(self, search): - # Need to return something to prevent index error - search.return_value = self.construct_search_response([1]) - results = self.get_results() - - results[10] # Performs search - - search.assert_any_call( - from_=10, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=1, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_slice_results(self, search): - search.return_value = self.construct_search_response([]) - results = self.get_results()[1:4] - - list(results) # Performs search - - search.assert_any_call( - from_=1, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=3, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_slice_results_multiple_times(self, search): - search.return_value = self.construct_search_response([]) - results = self.get_results()[10:][:10] - - list(results) # Performs search - - search.assert_any_call( - from_=10, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=10, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_slice_results_and_get_item(self, search): - # Need to return something to prevent index error - search.return_value = self.construct_search_response([1]) - results = self.get_results()[10:] - - results[10] # Performs search - - search.assert_any_call( - from_=20, - body={"query": "QUERY"}, - _source=False, - stored_fields="pk", - index="wagtail__searchtests_book", - size=1, - ) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_result_returned(self, search): - search.return_value = self.construct_search_response([1]) - results = self.get_results() - - self.assertEqual(results[0], models.Book.objects.get(id=1)) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_len_1(self, search): - search.return_value = self.construct_search_response([1]) - results = self.get_results() - - self.assertEqual(len(results), 1) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_len_2(self, search): - search.return_value = self.construct_search_response([1, 2]) - results = self.get_results() - - self.assertEqual(len(results), 2) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_duplicate_results(self, search): # Duplicates will not be removed - search.return_value = self.construct_search_response([1, 1]) - results = list( - self.get_results() - ) # Must cast to list so we only create one query - - self.assertEqual(len(results), 2) - self.assertEqual(results[0], models.Book.objects.get(id=1)) - self.assertEqual(results[1], models.Book.objects.get(id=1)) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_result_order(self, search): - search.return_value = self.construct_search_response([1, 2, 3]) - results = list( - self.get_results() - ) # Must cast to list so we only create one query - - self.assertEqual(results[0], models.Book.objects.get(id=1)) - self.assertEqual(results[1], models.Book.objects.get(id=2)) - self.assertEqual(results[2], models.Book.objects.get(id=3)) - - @mock.patch("elasticsearch.Elasticsearch.search") - def test_result_order_2(self, search): - search.return_value = self.construct_search_response([3, 2, 1]) - results = list( - self.get_results() - ) # Must cast to list so we only create one query - - self.assertEqual(results[0], models.Book.objects.get(id=3)) - self.assertEqual(results[1], models.Book.objects.get(id=2)) - self.assertEqual(results[2], models.Book.objects.get(id=1)) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 6, "Elasticsearch 6 required") -class TestElasticsearch6Mapping(TestCase): - fixtures = ["search"] - - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual( - json.dumps(a, sort_keys=True, default=default), - json.dumps(b, sort_keys=True, default=default), - ) - - def setUp(self): - # Create ES mapping - self.es_mapping = Elasticsearch6SearchBackend.mapping_class(models.Book) - - # Create ES document - self.obj = models.Book.objects.get(id=4) - - def test_get_document_type(self): - self.assertEqual(self.es_mapping.get_document_type(), "doc") - - def test_get_mapping(self): - # Build mapping - mapping = self.es_mapping.get_mapping() - - # Check - expected_result = { - "doc": { - "properties": { - "pk": {"type": "keyword", "store": True}, - "content_type": {"type": "keyword"}, - "_all_text": {"type": "text"}, - "_all_text_boost_2_0": {"type": "text"}, - "_edgengrams": { - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - "type": "text", - }, - "title": { - "type": "text", - "copy_to": ["_all_text", "_all_text_boost_2_0"], - }, - "title_edgengrams": { - "type": "text", - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - }, - "title_filter": {"type": "keyword"}, - "authors": { - "type": "nested", - "properties": { - "name": {"type": "text", "copy_to": "_all_text"}, - "name_edgengrams": { - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - "type": "text", - }, - "date_of_birth_filter": {"type": "date"}, - }, - }, - "authors_filter": {"type": "integer"}, - "publication_date_filter": {"type": "date"}, - "number_of_pages_filter": {"type": "integer"}, - "tags": { - "type": "nested", - "properties": { - "name": {"type": "text", "copy_to": "_all_text"}, - "slug_filter": {"type": "keyword"}, - }, - }, - "tags_filter": {"type": "integer"}, - } - } - } - - self.assertDictEqual(mapping, expected_result) - - def test_get_document_id(self): - self.assertEqual(self.es_mapping.get_document_id(self.obj), str(self.obj.pk)) - - def test_get_document(self): - # Get document - document = self.es_mapping.get_document(self.obj) - - # Sort edgengrams - if "_edgengrams" in document: - document["_edgengrams"].sort() - - # Check - expected_result = { - "pk": "4", - "content_type": ["searchtests.Book"], - "_edgengrams": [ - "J. R. R. Tolkien", - "The Fellowship of the Ring", - ], - "title": "The Fellowship of the Ring", - "title_edgengrams": "The Fellowship of the Ring", - "title_filter": "The Fellowship of the Ring", - "authors": [ - { - "name": "J. R. R. Tolkien", - "name_edgengrams": "J. R. R. Tolkien", - "date_of_birth_filter": datetime.date(1892, 1, 3), - } - ], - "authors_filter": [2], - "publication_date_filter": datetime.date(1954, 7, 29), - "number_of_pages_filter": 423, - "tags": [], - "tags_filter": [], - } - - self.assertDictEqual(document, expected_result) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 6, "Elasticsearch 6 required") -class TestElasticsearch6MappingInheritance(TestCase): - fixtures = ["search"] - - def assertDictEqual(self, a, b): - default = JSONSerializer().default - self.assertEqual( - json.dumps(a, sort_keys=True, default=default), - json.dumps(b, sort_keys=True, default=default), - ) - - def setUp(self): - # Create ES mapping - self.es_mapping = Elasticsearch6SearchBackend.mapping_class(models.Novel) - - self.obj = models.Novel.objects.get(id=4) - - def test_get_document_type(self): - self.assertEqual(self.es_mapping.get_document_type(), "doc") - - def test_get_mapping(self): - # Build mapping - mapping = self.es_mapping.get_mapping() - - # Check - expected_result = { - "doc": { - "properties": { - # New - "searchtests_novel__setting": { - "type": "text", - "copy_to": "_all_text", - }, - "searchtests_novel__setting_edgengrams": { - "type": "text", - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - }, - "searchtests_novel__protagonist": { - "type": "nested", - "properties": { - "name": { - "type": "text", - "copy_to": ["_all_text", "_all_text_boost_0_5"], - }, - "novel_id_filter": {"type": "integer"}, - }, - }, - "searchtests_novel__protagonist_id_filter": {"type": "integer"}, - "searchtests_novel__characters": { - "type": "nested", - "properties": { - "name": { - "type": "text", - "copy_to": ["_all_text", "_all_text_boost_0_25"], - } - }, - }, - # Inherited - "pk": {"type": "keyword", "store": True}, - "content_type": {"type": "keyword"}, - "_all_text": {"type": "text"}, - "_all_text_boost_0_25": {"type": "text"}, - "_all_text_boost_0_5": {"type": "text"}, - "_all_text_boost_2_0": {"type": "text"}, - "_edgengrams": { - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - "type": "text", - }, - "title": { - "type": "text", - "copy_to": ["_all_text", "_all_text_boost_2_0"], - }, - "title_edgengrams": { - "type": "text", - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - }, - "title_filter": {"type": "keyword"}, - "authors": { - "type": "nested", - "properties": { - "name": {"type": "text", "copy_to": "_all_text"}, - "name_edgengrams": { - "analyzer": "edgengram_analyzer", - "search_analyzer": "standard", - "type": "text", - }, - "date_of_birth_filter": {"type": "date"}, - }, - }, - "authors_filter": {"type": "integer"}, - "publication_date_filter": {"type": "date"}, - "number_of_pages_filter": {"type": "integer"}, - "tags": { - "type": "nested", - "properties": { - "name": {"type": "text", "copy_to": "_all_text"}, - "slug_filter": {"type": "keyword"}, - }, - }, - "tags_filter": {"type": "integer"}, - } - } - } - - self.assertDictEqual(mapping, expected_result) - - def test_get_document_id(self): - # This must be tests_searchtest instead of 'tests_searchtest_tests_searchtestchild' - # as it uses the contents base content type name. - # This prevents the same object being accidentally indexed twice. - self.assertEqual(self.es_mapping.get_document_id(self.obj), str(self.obj.pk)) - - def test_get_document(self): - # Build document - document = self.es_mapping.get_document(self.obj) - - # Sort edgengrams - if "_edgengrams" in document: - document["_edgengrams"].sort() - - # Sort characters - if "searchtests_novel__characters" in document: - document["searchtests_novel__characters"].sort(key=lambda c: c["name"]) - - # Check - expected_result = { - # New - "searchtests_novel__setting": "Middle Earth", - "searchtests_novel__setting_edgengrams": "Middle Earth", - "searchtests_novel__protagonist": { - "name": "Frodo Baggins", - "novel_id_filter": 4, - }, - "searchtests_novel__protagonist_id_filter": 8, - "searchtests_novel__characters": [ - {"name": "Bilbo Baggins"}, - {"name": "Frodo Baggins"}, - {"name": "Gandalf"}, - ], - # Changed - "content_type": ["searchtests.Novel", "searchtests.Book"], - "_edgengrams": [ - "J. R. R. Tolkien", - "Middle Earth", - "The Fellowship of the Ring", - ], - # Inherited - "pk": "4", - "title": "The Fellowship of the Ring", - "title_edgengrams": "The Fellowship of the Ring", - "title_filter": "The Fellowship of the Ring", - "authors": [ - { - "name": "J. R. R. Tolkien", - "name_edgengrams": "J. R. R. Tolkien", - "date_of_birth_filter": datetime.date(1892, 1, 3), - } - ], - "authors_filter": [2], - "publication_date_filter": datetime.date(1954, 7, 29), - "number_of_pages_filter": 423, - "tags": [], - "tags_filter": [], - } - - self.assertDictEqual(document, expected_result) - - -@unittest.skipIf(ELASTICSEARCH_VERSION[0] != 6, "Elasticsearch 6 required") -@mock.patch("wagtail.search.backends.elasticsearch5.Elasticsearch") -class TestBackendConfiguration(TestCase): - def test_default_settings(self, Elasticsearch): - Elasticsearch6SearchBackend(params={}) - - Elasticsearch.assert_called_with( - hosts=[ - { - "host": "localhost", - "port": 9200, - "url_prefix": "", - "use_ssl": False, - "verify_certs": False, - "http_auth": None, - } - ], - timeout=10, - ) - - def test_hosts(self, Elasticsearch): - Elasticsearch6SearchBackend( - params={ - "HOSTS": [ - { - "host": "127.0.0.1", - "port": 9300, - "use_ssl": True, - "verify_certs": True, - } - ] - } - ) - - Elasticsearch.assert_called_with( - hosts=[ - { - "host": "127.0.0.1", - "port": 9300, - "use_ssl": True, - "verify_certs": True, - } - ], - timeout=10, - ) - - def test_urls(self, Elasticsearch): - # This test backwards compatibility with old URLS setting - Elasticsearch6SearchBackend( - params={ - "URLS": [ - "http://localhost:12345", - "https://127.0.0.1:54321", - "http://username:password@elasticsearch.mysite.com", - "https://elasticsearch.mysite.com/hello", - ], - } - ) - - Elasticsearch.assert_called_with( - hosts=[ - { - "host": "localhost", - "port": 12345, - "url_prefix": "", - "use_ssl": False, - "verify_certs": False, - "http_auth": None, - }, - { - "host": "127.0.0.1", - "port": 54321, - "url_prefix": "", - "use_ssl": True, - "verify_certs": True, - "http_auth": None, - }, - { - "host": "elasticsearch.mysite.com", - "port": 80, - "url_prefix": "", - "use_ssl": False, - "verify_certs": False, - "http_auth": ("username", "password"), - }, - { - "host": "elasticsearch.mysite.com", - "port": 443, - "url_prefix": "/hello", - "use_ssl": True, - "verify_certs": True, - "http_auth": None, - }, - ], - timeout=10, - ) diff --git a/wagtail/test/settings.py b/wagtail/test/settings.py index de693b6bb4..7686867e44 100644 --- a/wagtail/test/settings.py +++ b/wagtail/test/settings.py @@ -224,10 +224,6 @@ if "ELASTICSEARCH_URL" in os.environ: backend = "wagtail.search.backends.elasticsearch8" elif os.environ.get("ELASTICSEARCH_VERSION") == "7": backend = "wagtail.search.backends.elasticsearch7" - elif os.environ.get("ELASTICSEARCH_VERSION") == "6": - backend = "wagtail.search.backends.elasticsearch6" - elif os.environ.get("ELASTICSEARCH_VERSION") == "5": - backend = "wagtail.search.backends.elasticsearch5" WAGTAILSEARCH_BACKENDS["elasticsearch"] = { "BACKEND": backend,