add file type validation

conf/base.py

@@ -201,6 +201,9 @@
         "image/tiff",
     ),
 )
+ACCEPTED_FILE_UPLOAD_EXTENSIONS = env.list(
+    "ACCEPTED_FILE_UPLOAD_EXTENSIONS", default=["pdf", "doc", "docx", "jpg", "pnf", "odt", "txt"]
+)
 # AV is performed by the API, but these empty settings are required by django-chunk-s3-av-upload-handlers
 CLAM_AV_USERNAME = env.str("CLAM_AV_USERNAME", "")
 CLAM_AV_PASSWORD = env.str("CLAM_AV_PASSWORD", "")

core/file_handler.py

@@ -1,6 +1,5 @@
 import boto3
 import logging
-import magic
 
 from django.conf import settings
 from django.core.files.uploadhandler import UploadFileException
@@ -47,24 +46,9 @@ def s3_client():
 
 class SafeS3FileUploadHandler(S3FileUploadHandler):
     """
-    S3FileUploadHandler with mime-type validation.
+    S3FileUploadHandler.
     """
 
-    ACCEPTED_FILE_UPLOAD_MIME_TYPES = settings.ACCEPTED_FILE_UPLOAD_MIME_TYPES
-
-    def receive_data_chunk(self, raw_data, start):
-        """
-        Receive a single file chunk from the browser, validate the
-        file type for the first chunk and leave the rest to super.
-        """
-        # For the first chunk
-        if start == 0:
-            mime = magic.from_buffer(raw_data, mime=True)
-            if mime not in self.ACCEPTED_FILE_UPLOAD_MIME_TYPES:
-                self.abort()
-                raise UnacceptableMimeTypeError(f"Unsupported file type: {mime}")
-        super().receive_data_chunk(raw_data, start)
-
     def file_complete(self, *args, **kwargs):
         """Override `file_complete` to ensure that all necessary attributes
         are set on the file object.

exporter/applications/forms/parties.py

@@ -7,10 +7,13 @@
 from django.core.validators import MaxLengthValidator, URLValidator
 from django.urls import reverse_lazy
 
+
 from core.common.forms import BaseForm
 from core.forms.layouts import ConditionalRadios, ConditionalRadiosQuestion
 from core.forms.widgets import Autocomplete
 from exporter.core.constants import CaseTypes, FileUploadFileTypes
+from exporter.core.validators import FileExtensionValidator
+
 from exporter.core.services import get_countries
 from lite_content.lite_exporter_frontend import strings
 from lite_content.lite_exporter_frontend.applications import PartyForm, PartyTypeForm
@@ -427,12 +430,15 @@ def get_title(self):
 
 class PartyDocumentUploadForm(forms.Form):
     title = "Upload an end-user document"
+    # from django.core.validators import FileExtensionValidator
     party_document = forms.FileField(
         label=FileUploadFileTypes.UPLOAD_GUIDANCE_TEXT,
         error_messages={
             "required": "Select an end-user document",
         },
+        validators=[FileExtensionValidator()],
     )
+
     product_differences_note = forms.CharField(
         widget=forms.Textarea(attrs={"rows": "5"}),
         label="Describe any differences between products listed in the document and products on the application (optional)",
@@ -491,6 +497,7 @@ class PartyEnglishTranslationDocumentUploadForm(forms.Form):
         error_messages={
             "required": "Select an English translation",
         },
+        validators=[FileExtensionValidator()],
     )
 
     def __init__(self, edit, *args, **kwargs):
@@ -519,6 +526,7 @@ class PartyCompanyLetterheadDocumentUploadForm(forms.Form):
         error_messages={
             "required": "Select a document on company letterhead",
         },
+        validators=[FileExtensionValidator()],
     )
 
     def __init__(self, edit, *args, **kwargs):
@@ -542,7 +550,7 @@ def get_title(self):
 
 class PartyEC3DocumentUploadForm(forms.Form):
     title = "Upload an EC3 form (optional)"
-    party_ec3_document = forms.FileField(label="", required=False)
+    party_ec3_document = forms.FileField(label="", required=False, validators=[FileExtensionValidator()])
     ec3_missing_reason = forms.CharField(
         widget=forms.Textarea(attrs={"rows": "5"}),
         label="",

exporter/core/validators.py

@@ -3,6 +3,8 @@
 
 from django.core.exceptions import ValidationError
 from django.utils import timezone
+from django.conf import settings
+from django.core.validators import FileExtensionValidator
 
 from exporter.applications.helpers.date_fields import format_date
 
@@ -52,3 +54,9 @@ def __init__(self, message, **kwargs):
     def __call__(self, value):
         if value > (date.today() + self.relativedelta):
             raise ValidationError(self.message)
+
+
+class FileExtensionValidator(FileExtensionValidator):
+    def __init__(self, allowed_extensions=[], message=None):
+        self.message = message or "The file type is not supported. Upload a supported file type"
+        self.allowed_extensions = allowed_extensions or settings.ACCEPTED_FILE_UPLOAD_EXTENSIONS

exporter/goods/forms/common.py

@@ -23,7 +23,7 @@
     get_pv_gradings_v2,
     get_units,
 )
-from exporter.core.validators import PastDateValidator
+from exporter.core.validators import FileExtensionValidator, PastDateValidator
 from exporter.core.constants import ProductSecurityFeatures, FileUploadFileTypes
 
 
@@ -363,6 +363,7 @@ class Layout:
         error_messages={
             "required": "Select a document that shows what your product is designed to do",
         },
+        validators=[FileExtensionValidator()],
     )
     description = forms.CharField(
         widget=forms.Textarea(attrs={"rows": "5"}),

exporter/goods/forms/firearms.py

@@ -22,6 +22,7 @@
     PotentiallyUnsafeClearableFileInput,
 )
 from exporter.core.validators import (
+    FileExtensionValidator,
     FutureDateValidator,
     PastDateValidator,
     RelativeDeltaDateValidator,
@@ -223,6 +224,7 @@ class Layout:
             "required": "Select a registered firearms dealer certificate",
         },
         widget=PotentiallyUnsafeClearableFileInput,
+        validators=[FileExtensionValidator()],
     )
 
     reference_code = forms.CharField(
@@ -333,6 +335,7 @@ class Layout:
         widget=PotentiallyUnsafeClearableFileInput(
             force_required=True,
         ),
+        validators=[FileExtensionValidator()],
     )
 
     section_certificate_number = forms.CharField(

exporter/goods/forms/goods.py

@@ -22,6 +22,7 @@
 )
 from exporter.core.helpers import convert_control_list_entries, str_to_bool, convert_control_list_entries_to_options
 from exporter.core.services import get_control_list_entries, get_pv_gradings, get_units
+from exporter.core.validators import FileExtensionValidator
 from exporter.goods.helpers import get_category_display_string, good_summary
 from core.common.forms import BaseForm
 from lite_content.lite_exporter_frontend.goods import (
@@ -1112,6 +1113,7 @@ class AttachFirearmsDealerCertificateForm(forms.Form):
         error_messages={
             "required": "Select certificate file to upload",
         },
+        validators=[FileExtensionValidator()],
     )
 
     reference_code = forms.CharField(

exporter/organisation/forms.py

@@ -9,6 +9,7 @@
 from crispy_forms_gds.layout import Submit, Layout, HTML
 
 from exporter.core.constants import FileUploadFileTypes
+from exporter.core.validators import FileExtensionValidator
 
 
 def validate_expiry_date(value):
@@ -63,6 +64,7 @@ class Layout:
         label=FileUploadFileTypes.UPLOAD_GUIDANCE_TEXT,
         help_text="The file must be smaller than 50MB",
         error_messages={"required": "Select certificate file to upload"},
+        validators=[FileExtensionValidator()],
     )
     reference_code = forms.CharField(
         label="Certificate number",

unit_tests/exporter/applications/forms/test_parties.py

@@ -307,11 +307,18 @@ def test_party_documents_form(data, valid, errors):
     (
         (
             {"description": "test", "document_in_english": "True", "document_on_letterhead": "True"},
-            {"party_document": SimpleUploadedFile("test", b"test_content")},
+            {"party_document": SimpleUploadedFile("test.pdf", b"test_content")},
             False,
             True,
             None,
         ),
+        (
+            {"description": "test", "document_in_english": "True", "document_on_letterhead": "True"},
+            {"party_document": SimpleUploadedFile("test.obs", b"test_content")},
+            False,
+            False,
+            {"party_document": ["The file type is not supported. Upload a supported file type"]},
+        ),
         (
             {"description": "test", "document_in_english": "True", "document_on_letterhead": "True"},
             {},
@@ -361,11 +368,18 @@ def test_party_document_upload_form(data, files, edit, valid, errors):
     (
         (
             {},
-            {"party_eng_translation_document": SimpleUploadedFile("test", b"test_content")},
+            {"party_eng_translation_document": SimpleUploadedFile("test.pdf", b"test_content")},
             False,
             True,
             None,
         ),
+        (
+            {},
+            {"party_eng_translation_document": SimpleUploadedFile("test.pdd", b"test_content")},
+            False,
+            False,
+            {"party_eng_translation_document": ["The file type is not supported. Upload a supported file type"]},
+        ),
         (
             {},
             {},
@@ -398,11 +412,18 @@ def test_party_english_translation_document_upload_form(data, files, edit, valid
     (
         (
             {},
-            {"party_letterhead_document": SimpleUploadedFile("test", b"test_content")},
+            {"party_letterhead_document": SimpleUploadedFile("test.pdf", b"test_content")},
             False,
             True,
             None,
         ),
+        (
+            {},
+            {"party_letterhead_document": SimpleUploadedFile("test.ioi", b"test_content")},
+            False,
+            False,
+            {"party_letterhead_document": ["The file type is not supported. Upload a supported file type"]},
+        ),
         (
             {},
             {},

unit_tests/exporter/applications/views/goods/component/test_add_good_component.py

@@ -178,7 +178,7 @@ def test_add_good_component_accessory_end_to_end(
 
     response = post_to_step(
         AddGoodComponentSteps.PRODUCT_DOCUMENT_UPLOAD,
-        {"product_document": SimpleUploadedFile("data sheet", b"This is a detailed spec of this Rifle")},
+        {"product_document": SimpleUploadedFile("data_sheet.pdf", b"This is a detailed spec of this Rifle")},
     )
 
     assert response.status_code == 200
@@ -227,7 +227,7 @@ def test_add_good_component_accessory_end_to_end(
     }
     assert post_good_document_matcher.called_once
     assert post_good_document_matcher.last_request.json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": ""}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": ""}
     ]
 
 

unit_tests/exporter/applications/views/goods/component/test_edit_component.py

@@ -35,7 +35,7 @@ def good_on_application(data_standard_case):
 @pytest.fixture
 def product_document():
     return {
-        "product_document": SimpleUploadedFile("data sheet", b"This is a detailed spec of this Rifle"),
+        "product_document": SimpleUploadedFile("data_sheet.pdf", b"This is a detailed spec of this Rifle"),
         "description": "product data sheet",
     }
 
@@ -474,7 +474,7 @@ def test_edit_product_document_availability_upload_new_document(
     assert document_delete_request.matcher.called_once
 
     assert requests_mock.request_history.pop().json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": "product data sheet"}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": "product data sheet"}
     ]
 
     assert requests_mock.request_history.pop().json() == {
@@ -520,7 +520,7 @@ def test_upload_new_product_document_to_replace_existing_one(
     assert document_delete_request.matcher.called_once
 
     assert requests_mock.request_history.pop().json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": "product data sheet"}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": "product data sheet"}
     ]
 
     assert requests_mock.request_history.pop().json() == {"is_document_sensitive": False}
@@ -567,5 +567,5 @@ def test_edit_product_document_upload_form(
     assert document_delete_request.matcher.called_once
 
     assert requests_mock.last_request.json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": "product data sheet"}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": "product data sheet"}
     ]

unit_tests/exporter/applications/views/goods/firearm/test_add_good_firearm.py

@@ -255,7 +255,7 @@ def test_add_good_firearm_product_document_available_but_not_sensitive(
 
     response = post_to_step(
         AddGoodFirearmSteps.PRODUCT_DOCUMENT_UPLOAD,
-        {"product_document": SimpleUploadedFile("data sheet", b"This is a detailed spec of this Rifle")},
+        {"product_document": SimpleUploadedFile("data_sheet.pdf", b"This is a detailed spec of this Rifle")},
     )
     assert response.status_code == 200
     assert isinstance(response.context["form"], FirearmCategoryForm)
@@ -452,7 +452,7 @@ def test_add_good_firearm_with_rfd_document_submission(
     )
     response = post_to_step(
         AddGoodFirearmSteps.PRODUCT_DOCUMENT_UPLOAD,
-        {"product_document": SimpleUploadedFile("data sheet", b"This is a detailed spec of this Rifle")},
+        {"product_document": SimpleUploadedFile("data_sheet.pdf", b"This is a detailed spec of this Rifle")},
     )
 
     assert response.status_code == 302
@@ -497,7 +497,9 @@ def test_add_good_firearm_with_rfd_document_submission(
 
     assert post_good_document_matcher.called_once
     good_doc_request = post_good_document_matcher.last_request
-    assert good_doc_request.json() == [{"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": ""}]
+    assert good_doc_request.json() == [
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": ""}
+    ]
 
     assert post_applications_document_matcher.called_once
     application_doc_request = post_applications_document_matcher.last_request
@@ -701,7 +703,7 @@ def test_add_good_firearm_without_rfd_document_submission_registered_firearms_de
         {"is_covered_by_section_5": "no"},
     )
     expiry_date = datetime.date.today() + datetime.timedelta(days=5)
-    file_name = "test"
+    file_name = "test.pdf"
     post_to_step(
         AddGoodFirearmSteps.ATTACH_RFD_CERTIFICATE,
         {

unit_tests/exporter/applications/views/goods/firearm/test_edit_firearm.py

@@ -70,7 +70,7 @@ def edit_product_sensitivity_url(application, good_on_application):
 @pytest.fixture(autouse=True)
 def product_document():
     return {
-        "product_document": SimpleUploadedFile("data sheet", b"This is a detailed spec of this Rifle"),
+        "product_document": SimpleUploadedFile("data_sheet.pdf", b"This is a detailed spec of this Rifle"),
         "description": "product data sheet",
     }
 
@@ -302,7 +302,7 @@ def test_edit_product_document_upload_form(
     assert document_delete_request.matcher.called_once
 
     assert requests_mock.last_request.json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": "product data sheet"}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": "product data sheet"}
     ]
 
 
@@ -353,7 +353,7 @@ def test_upload_new_product_document_to_replace_existing_one(
     assert document_delete_request.matcher.called_once
 
     assert requests_mock.request_history.pop().json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": "product data sheet"}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": "product data sheet"}
     ]
 
     assert requests_mock.request_history.pop().json() == {"is_document_sensitive": False}
@@ -443,7 +443,7 @@ def test_edit_product_document_availability_upload_new_document(
     assert document_delete_request.matcher.called_once
 
     assert requests_mock.request_history.pop().json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": "product data sheet"}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": "product data sheet"}
     ]
 
     assert requests_mock.request_history.pop().json() == {

unit_tests/exporter/applications/views/goods/material/test_add_good_material.py

@@ -152,7 +152,7 @@ def test_add_good_material_end_to_end(
 
     response = post_to_step(
         AddGoodMaterialSteps.PRODUCT_DOCUMENT_UPLOAD,
-        {"product_document": SimpleUploadedFile("data sheet", b"This is a detailed spec of this Rifle")},
+        {"product_document": SimpleUploadedFile("data_sheet.pdf", b"This is a detailed spec of this Rifle")},
     )
 
     assert response.status_code == 200
@@ -199,7 +199,7 @@ def test_add_good_material_end_to_end(
 
     assert post_good_document_matcher.called_once
     assert post_good_document_matcher.last_request.json() == [
-        {"name": "data sheet", "s3_key": "data sheet", "size": 0, "description": ""}
+        {"name": "data_sheet.pdf", "s3_key": "data_sheet.pdf", "size": 0, "description": ""}
     ]