diff --git a/django/db/models/sql/query.py b/django/db/models/sql/query.py
index df654052fb..ba6a50a6f8 100644
--- a/django/db/models/sql/query.py
+++ b/django/db/models/sql/query.py
@@ -1643,6 +1643,11 @@ class Query(object):
         for item in ordering:
             if not hasattr(item, 'resolve_expression') and not ORDER_PATTERN.match(item):
                 errors.append(item)
+            if getattr(item, 'contains_aggregate', False):
+                raise FieldError(
+                    'Using an aggregate in order_by() without also including '
+                    'it in annotate() is not allowed: %s' % item
+                )
         if errors:
             raise FieldError('Invalid order_by arguments: %s' % errors)
         if ordering:
diff --git a/tests/aggregation/tests.py b/tests/aggregation/tests.py
index 10f1275cf6..b98764537f 100644
--- a/tests/aggregation/tests.py
+++ b/tests/aggregation/tests.py
@@ -105,6 +105,14 @@ class AggregateTestCase(TestCase):
     def test_empty_aggregate(self):
         self.assertEqual(Author.objects.all().aggregate(), {})
 
+    def test_aggregate_in_order_by(self):
+        msg = (
+            'Using an aggregate in order_by() without also including it in '
+            'annotate() is not allowed: Avg(F(book__rating)'
+        )
+        with self.assertRaisesMessage(FieldError, msg):
+            Author.objects.values('age').order_by(Avg('book__rating'))
+
     def test_single_aggregate(self):
         vals = Author.objects.aggregate(Avg("age"))
         self.assertEqual(vals, {"age__avg": Approximate(37.4, places=1)})