Skip to content

Commit

Permalink
Merge pull request #307 from City-of-Helsinki/issue-266
Browse files Browse the repository at this point in the history
Add upload endpoint v1/file to allow file uploads linked to a section
  • Loading branch information
Rikuoja authored Apr 12, 2019
2 parents 67b4378 + bd75a4a commit bf72d74
Show file tree
Hide file tree
Showing 22 changed files with 956 additions and 29 deletions.
71 changes: 71 additions & 0 deletions democracy/migrations/0041_add_file_models.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.10 on 2018-02-22 13:46
from __future__ import unicode_literals

from django.conf import settings
import django.core.files.storage
from django.db import migrations, models
import django.db.models.deletion
import django.utils.timezone
import parler.models


class Migration(migrations.Migration):

dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('democracy', '0040_add_hearing_project_phase'),
]

operations = [
migrations.CreateModel(
name='SectionFile',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(db_index=True, default=django.utils.timezone.now, editable=False, verbose_name='time of creation')),
('modified_at', models.DateTimeField(default=django.utils.timezone.now, editable=False, verbose_name='time of last modification')),
('published', models.BooleanField(db_index=True, default=True, verbose_name='public')),
('deleted', models.BooleanField(db_index=True, default=False, editable=False, verbose_name='deleted')),
('uploaded_file', models.FileField(storage=django.core.files.storage.FileSystemStorage(location='/home/heikkjus/code/helsinki/kerrokantasi/var/protected'), upload_to='files/%Y/%m', verbose_name='file')),
('ordering', models.IntegerField(db_index=True, default=1, help_text='The ordering position for this object. Objects with smaller numbers appear first.', verbose_name='ordering')),
('created_by', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='sectionfile_created', to=settings.AUTH_USER_MODEL, verbose_name='created by')),
('modified_by', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='sectionfile_modified', to=settings.AUTH_USER_MODEL, verbose_name='last modified by')),
('section', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='democracy.Section')),
],
options={
'verbose_name': 'section file',
'verbose_name_plural': 'section files',
'ordering': ('ordering',),
},
bases=(parler.models.TranslatableModelMixin, models.Model),
),
migrations.CreateModel(
name='SectionFileTranslation',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('language_code', models.CharField(db_index=True, max_length=15, verbose_name='Language')),
('title', models.CharField(blank=True, default='', max_length=255, verbose_name='title')),
('caption', models.TextField(blank=True, default='', verbose_name='caption')),
('master', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='democracy.SectionFile')),
],
options={
'verbose_name': 'section file Translation',
'db_tablespace': '',
'default_permissions': (),
'managed': True,
'db_table': 'democracy_sectionfile_translation',
},
),
migrations.AlterModelOptions(
name='contactperson',
options={'ordering': ['name'], 'verbose_name': 'contact person', 'verbose_name_plural': 'contact persons'},
),
migrations.AlterModelOptions(
name='sectionimage',
options={'ordering': ('ordering',), 'verbose_name': 'section image', 'verbose_name_plural': 'section images'},
),
migrations.AlterUniqueTogether(
name='sectionfiletranslation',
unique_together=set([('language_code', 'master')]),
),
]
21 changes: 21 additions & 0 deletions democracy/migrations/0042_sectionfile_section_nullable.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.10 on 2018-02-22 14:55
from __future__ import unicode_literals

from django.db import migrations, models
import django.db.models.deletion


class Migration(migrations.Migration):

dependencies = [
('democracy', '0041_add_file_models'),
]

operations = [
migrations.AlterField(
model_name='sectionfile',
name='section',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='files', to='democracy.Section'),
),
]
21 changes: 21 additions & 0 deletions democracy/migrations/0043_basefile_filename_max_len.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.11.15 on 2018-10-26 09:28
from __future__ import unicode_literals

import django.core.files.storage
from django.db import migrations, models


class Migration(migrations.Migration):

dependencies = [
('democracy', '0042_sectionfile_section_nullable'),
]

operations = [
migrations.AlterField(
model_name='sectionfile',
name='uploaded_file',
field=models.FileField(max_length=2048, storage=django.core.files.storage.FileSystemStorage(location='/home/heikkjus/code/helsinki/kerrokantasi/var/protected'), upload_to='files/%Y/%m', verbose_name='file'),
),
]
3 changes: 2 additions & 1 deletion democracy/models/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from .hearing import Hearing
from .label import Label
from .section import Section, SectionComment, SectionImage, SectionType
from .section import Section, SectionComment, SectionImage, SectionFile, SectionType
from .section import SectionPoll, SectionPollOption, SectionPollAnswer
from .organization import ContactPerson, Organization
from .project import Project, ProjectPhase
Expand All @@ -19,4 +19,5 @@
"Organization",
"Project",
"ProjectPhase",
"SectionFile",
]
20 changes: 20 additions & 0 deletions democracy/models/files.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
from django.db import models
from django.db.models import FileField
from django.utils.translation import ugettext_lazy as _
from django.core.files.storage import FileSystemStorage
from django.conf import settings

from .base import ORDERING_HELP, BaseModel

protected_storage = FileSystemStorage(location=settings.SENDFILE_ROOT)


class BaseFile(BaseModel):
uploaded_file = FileField(
verbose_name=_('file'), max_length=2048, upload_to='files/%Y/%m', storage=protected_storage
)
ordering = models.IntegerField(verbose_name=_('ordering'), default=1, db_index=True, help_text=ORDERING_HELP)

class Meta:
abstract = True
ordering = ("ordering")
51 changes: 50 additions & 1 deletion democracy/models/section.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
import logging
import re
from django.core.urlresolvers import get_resolver
from django.db import models
from django.utils.translation import ugettext_lazy as _
from reversion import revisions
Expand All @@ -8,6 +11,7 @@
from democracy.models.comment import BaseComment, recache_on_save
from democracy.models.poll import BasePoll, BasePollOption, BasePollAnswer, poll_option_recache_on_save
from democracy.models.images import BaseImage
from democracy.models.files import BaseFile
from democracy.plugins import get_implementation

from democracy.enums import InitialSectionType
Expand All @@ -18,6 +22,8 @@

INITIAL_SECTION_TYPE_IDS = set(value for key, value in InitialSectionType.__dict__.items() if key[:1] != '_')

LOG = logging.getLogger(__name__)


class SectionTypeQuerySet(models.QuerySet):
def initial(self):
Expand Down Expand Up @@ -74,7 +80,32 @@ def save(self, *args, **kwargs):
# This is a new section or changing type from closure info,
# automatically derive next ordering, if possible
self.ordering = max(self.hearing.sections.values_list("ordering", flat=True) or [0]) + 1
return super(Section, self).save(*args, **kwargs)
obj = super(Section, self).save(*args, **kwargs)
self.claim_orphan_files()
return obj

def claim_orphan_files(self):
"""
While creating the Section files might have been uploaded that should be
linked to the Section. Try to find such SectionFile objects and
link them.
CKEditor creates plain old <a> elements for uploaded files. We have to
parse the section content to figure out if there are new files.
"""
# get regex pattern of protected sectionfile endpoint
resolver = get_resolver(None)
url = resolver.reverse_dict.getlist('serve_file')
if not url:
LOG.error('serve_file URL pattern not found')
return 0
pattern = url[0][1].rstrip('$')

sectionfile_pks = []
for translation in self.translations.all():
for match in re.finditer(pattern, translation.content):
sectionfile_pks.append(match.groupdict()['pk'])
return SectionFile.objects.filter(section__isnull=True, pk__in=sectionfile_pks).update(section_id=self.pk)

def check_commenting(self, request):
super().check_commenting(request)
Expand Down Expand Up @@ -104,6 +135,24 @@ class Meta:
ordering = ('ordering',)


class SectionFile(BaseFile, TranslatableModel):
parent_field = "section"
section = models.ForeignKey(Section, related_name="files", blank=True, null=True)
translations = TranslatedFields(
title=models.CharField(verbose_name=_('title'), max_length=255, blank=True, default=''),
caption=models.TextField(verbose_name=_('caption'), blank=True, default=''),
)
objects = BaseModelManager.from_queryset(TranslatableQuerySet)()

class Meta:
verbose_name = _('section file')
verbose_name_plural = _('section files')
ordering = ('ordering',)

def __str__(self):
return '%s - %s' % (self.pk, self.uploaded_file.name)


@revisions.register
@recache_on_save
class SectionComment(BaseComment):
Expand Down
27 changes: 25 additions & 2 deletions democracy/tests/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@

import pytest
from django.contrib.auth import get_user_model
from django.core.files.base import ContentFile
from django.utils.timezone import now
from rest_framework.test import APIClient

from democracy.enums import Commenting, InitialSectionType
from democracy.factories.hearing import HearingFactory, LabelFactory
from democracy.models import ContactPerson, Hearing, Label, Project, ProjectPhase, Section, SectionType, Organization
from democracy.tests.utils import assert_ascending_sequence, create_default_images
from democracy.models import ContactPerson, Hearing, Label, Project, ProjectPhase, Section, SectionFile, SectionType, Organization
from democracy.tests.utils import FILES, assert_ascending_sequence, create_default_images, create_default_files, get_file_path


default_comment_content = 'I agree with you sir Lancelot. My favourite colour is blue'
Expand Down Expand Up @@ -71,6 +72,7 @@ def default_hearing(john_doe, contact_person, default_organization, default_proj
commenting=Commenting.OPEN
)
create_default_images(section)
create_default_files(section)
section.comments.create(created_by=john_doe, content=default_comment_content[::-1])
section.comments.create(created_by=john_doe, content=red_comment_content[::-1])
section.comments.create(created_by=john_doe, content=green_comment_content[::-1])
Expand Down Expand Up @@ -193,6 +195,17 @@ def admin_api_client(admin_user):
return api_client


@pytest.fixture()
def admin_api_client_logged_in(admin_user):
api_client = APIClient()
api_client.force_authenticate(user=admin_user)
api_client.user = admin_user
admin_user.set_password('foo')
admin_user.save()
api_client.login(username=admin_user.username, password='foo')
return api_client


@pytest.fixture()
def api_client():
return APIClient()
Expand Down Expand Up @@ -351,3 +364,13 @@ def geojson_featurecollection(geojson_point, geojson_polygon):
},
]
}


@pytest.fixture
def section_file_orphan():
section_file = SectionFile()
with open(get_file_path(FILES['TXT']), 'rb') as fp:
cf = ContentFile(fp.read())
section_file.uploaded_file.save('test/file.txt', cf, save=False)
section_file.save()
return section_file
1 change: 1 addition & 0 deletions democracy/tests/files/text_file.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A quick brown fox jumps over the lazy dog
Loading

0 comments on commit bf72d74

Please sign in to comment.