diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e2144d37fb..49bf5a77f4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -12,6 +12,7 @@ Changelog * Added `Page` methods `can_exist_under`, `can_create_at`, `can_move_to` for customising page type business rules * `wagtailadmin.utils.send_mail` now passes extra keyword arguments to Django's `send_mail` function (Matthew Downey) * `page_unpublish` signal is now fired for each page that was unpublished by a call to `PageQuerySet.unpublish()` + * Add `get_upload_to` method to `AbstractImage`, to allow overriding the default image upload path (Ben Emery) * Fix: HTTP cache purge now works again on Python 2 (Mitchel Cabuloy) * Fix: Locked pages can no longer be unpublished (Alex Bridge) * Fix: Site records now implement `get_by_natural_key` diff --git a/docs/releases/1.3.rst b/docs/releases/1.3.rst index da3c66123a..165554bcf6 100644 --- a/docs/releases/1.3.rst +++ b/docs/releases/1.3.rst @@ -22,6 +22,7 @@ Minor features * Added ``Page`` methods ``can_exist_under``, ``can_create_at``, ``can_move_to`` for customising page type business rules * ``wagtailadmin.utils.send_mail`` now passes extra keyword arguments to Django's ``send_mail`` function (Matthew Downey) * ``page_unpublish`` signal is now fired for each page that was unpublished by a call to ``PageQuerySet.unpublish()`` + * Add `get_upload_to` method to `AbstractImage`, to allow overriding the default image upload path (Ben Emery) Bug fixes diff --git a/wagtail/tests/testapp/migrations/0019_customimagefilepath.py b/wagtail/tests/testapp/migrations/0019_customimagefilepath.py new file mode 100644 index 0000000000..d729a3520f --- /dev/null +++ b/wagtail/tests/testapp/migrations/0019_customimagefilepath.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models, migrations +from django.conf import settings +import taggit.managers +import wagtail.wagtailadmin.taggable +import wagtail.wagtailimages.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('taggit', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('tests', '0018_singletonpage'), + ] + + operations = [ + migrations.CreateModel( + name='CustomImageFilePath', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('title', models.CharField(max_length=255, verbose_name='Title')), + ('file', models.ImageField(height_field='height', upload_to=wagtail.wagtailimages.models.get_upload_to, width_field='width', verbose_name='File')), + ('width', models.IntegerField(verbose_name='Width', editable=False)), + ('height', models.IntegerField(verbose_name='Height', editable=False)), + ('created_at', models.DateTimeField(db_index=True, auto_now_add=True, verbose_name='Created at')), + ('focal_point_x', models.PositiveIntegerField(null=True, blank=True)), + ('focal_point_y', models.PositiveIntegerField(null=True, blank=True)), + ('focal_point_width', models.PositiveIntegerField(null=True, blank=True)), + ('focal_point_height', models.PositiveIntegerField(null=True, blank=True)), + ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text=None, verbose_name='Tags')), + ('uploaded_by_user', models.ForeignKey(blank=True, editable=False, to=settings.AUTH_USER_MODEL, null=True, verbose_name='Uploaded by user')), + ('file_size', models.PositiveIntegerField(null=True, editable=False)), + ], + options={ + 'abstract': False, + }, + bases=(models.Model, wagtail.wagtailadmin.taggable.TagSearchable), + ), + ] diff --git a/wagtail/tests/testapp/models.py b/wagtail/tests/testapp/models.py index 0671f2dc01..633b3edf28 100644 --- a/wagtail/tests/testapp/models.py +++ b/wagtail/tests/testapp/models.py @@ -1,5 +1,8 @@ from __future__ import unicode_literals +import hashlib +import os + from django.db import models from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger from django.utils.encoding import python_2_unicode_compatible @@ -461,7 +464,6 @@ class StreamModel(models.Model): ('image', ImageChooserBlock()), ]) - class StreamPage(Page): body = StreamField([ ('text', CharBlock()), @@ -536,3 +538,34 @@ class GenericSnippetPage(Page): snippet_content_type = models.ForeignKey(ContentType, on_delete=models.SET_NULL, null=True) snippet_object_id = models.PositiveIntegerField(null=True) snippet_content_object = GenericForeignKey('snippet_content_type', 'snippet_object_id') + + +class CustomImageFilePath(AbstractImage): + def get_upload_to(self, filename): + """Create a path that's file-system friendly. + + By hashing the file's contents we guarantee an equal distribution + of files within our root directories. This also gives us a + better chance of uploading images with the same filename, but + different contents - this isn't guaranteed as we're only using + the first three characters of the checksum. + """ + original_filepath = super(CustomImageFilePath, self).get_upload_to(filename) + folder_name, filename = original_filepath.split(os.path.sep) + + # Ensure that we consume the entire file, we can't guarantee that + # the stream has not be partially (or entirely) consumed by + # another process + original_position = self.file.tell() + self.file.seek(0) + hash256 = hashlib.sha256() + + while True: + data = self.file.read(256) + if not data: + break + hash256.update(data) + checksum = hash256.hexdigest() + + self.file.seek(original_position) + return os.path.join(folder_name, checksum[:3], filename) diff --git a/wagtail/wagtailimages/models.py b/wagtail/wagtailimages/models.py index 0ca6ee1ab1..3c090c10c8 100644 --- a/wagtail/wagtailimages/models.py +++ b/wagtail/wagtailimages/models.py @@ -48,19 +48,9 @@ class ImageQuerySet(SearchableQuerySetMixin, models.QuerySet): def get_upload_to(instance, filename): - folder_name = 'original_images' - filename = instance.file.field.storage.get_valid_name(filename) + # Dumb proxy to instance method. + return instance.get_upload_to(filename) - # do a unidecode in the filename and then - # replace non-ascii characters in filename with _ , to sidestep issues with filesystem encoding - filename = "".join((i if ord(i) < 128 else '_') for i in unidecode(filename)) - - # Truncate filename so it fits in the 100 character limit - # https://code.djangoproject.com/ticket/9893 - while len(os.path.join(folder_name, filename)) >= 95: - prefix, dot, extension = filename.rpartition('.') - filename = prefix[:-1] + dot + extension - return os.path.join(folder_name, filename) @python_2_unicode_compatible @@ -106,6 +96,21 @@ class AbstractImage(models.Model, TagSearchable): return self.file_size + def get_upload_to(self, filename): + folder_name = 'original_images' + filename = self.file.field.storage.get_valid_name(filename) + + # do a unidecode in the filename and then + # replace non-ascii characters in filename with _ , to sidestep issues with filesystem encoding + filename = "".join((i if ord(i) < 128 else '_') for i in unidecode(filename)) + + # Truncate filename so it fits in the 100 character limit + # https://code.djangoproject.com/ticket/9893 + while len(os.path.join(folder_name, filename)) >= 95: + prefix, dot, extension = filename.rpartition('.') + filename = prefix[:-1] + dot + extension + return os.path.join(folder_name, filename) + def get_usage(self): return get_object_usage(self) diff --git a/wagtail/wagtailimages/tests/tests.py b/wagtail/wagtailimages/tests/tests.py index 5e1994b90c..a80c4f2707 100644 --- a/wagtail/wagtailimages/tests/tests.py +++ b/wagtail/wagtailimages/tests/tests.py @@ -7,7 +7,7 @@ from django.core.urlresolvers import reverse from taggit.forms import TagField, TagWidget -from wagtail.tests.testapp.models import CustomImage +from wagtail.tests.testapp.models import CustomImage, CustomImageFilePath from wagtail.tests.utils import WagtailTestUtils from wagtail.wagtailimages.utils import generate_signature, verify_signature from wagtail.wagtailimages.rect import Rect, Vector @@ -374,3 +374,21 @@ class TestRenditionFilenames(TestCase): rendition = image.get_rendition('fill-100x100') self.assertEqual(rendition.file.name, 'images/test_rf3.15ee4958.fill-100x100.png') + + +class TestDifferentUpload(TestCase): + def test_upload_path(self): + image = CustomImageFilePath.objects.create( + title="Test image", + file=get_test_image_file(), + ) + + second_image = CustomImageFilePath.objects.create( + title="Test Image", + file=get_test_image_file(colour='black'), + + ) + + # The files should be uploaded based on it's content, not just + # it's filename + self.assertFalse(image.file.url == second_image.file.url) diff --git a/wagtail/wagtailimages/tests/utils.py b/wagtail/wagtailimages/tests/utils.py index 6691d53c08..5426c34629 100644 --- a/wagtail/wagtailimages/tests/utils.py +++ b/wagtail/wagtailimages/tests/utils.py @@ -9,8 +9,8 @@ from wagtail.wagtailimages.models import get_image_model Image = get_image_model() -def get_test_image_file(filename='test.png'): +def get_test_image_file(filename='test.png', colour='white', size=(640, 480)): f = BytesIO() - image = PIL.Image.new('RGB', (640, 480), 'white') + image = PIL.Image.new('RGB', size, colour) image.save(f, 'PNG') return ImageFile(f, name=filename)