Merge pull request #2010 from uktrade/uat

PROD Release

api/applications/tests/factories.py

@@ -123,7 +123,7 @@ class DenialMatchFactory(factory.django.DjangoModelFactory):
     address = factory.LazyAttribute(lambda n: faker.address())
     notifying_government = factory.LazyAttribute(lambda n: faker.country())
     country = factory.LazyAttribute(lambda n: faker.country())
-    item_list_codes = factory.LazyAttribute(lambda n: faker.word())
+    denial_cle = factory.LazyAttribute(lambda n: faker.word())
     item_description = factory.LazyAttribute(lambda n: faker.sentence())
     consignee_name = factory.LazyAttribute(lambda n: faker.name())
     end_use = factory.LazyAttribute(lambda n: faker.sentence())

api/data_workspace/tests/test_external_data_views.py

@@ -28,7 +28,7 @@ def test_denial_view(self):
             "reference",
             "notifying_government",
             "country",
-            "item_list_codes",
+            "denial_cle",
             "item_description",
             "end_use",
             "data",

api/document_data/health_checks.py

@@ -63,10 +63,10 @@ def check_status(self):
         # If the date is in the past then it means the task that should have
         # run previously hasn't for some reason.
         backup_schedule = app.conf.beat_schedule[BACKUP_DOCUMENT_DATA_SCHEDULE_NAME]["schedule"]
-        next_run_delta = backup_schedule.remaining_estimate(latest_backup_log.ended_at)
-        now = timezone.now()
+        next_run_delta = backup_schedule.remaining_estimate(timezone.localtime(latest_backup_log.ended_at))
+        now = timezone.localtime()
         next_run = now + next_run_delta
-        if next_run < timezone.now():
+        if next_run < now:
             raise BackupDocumentDataHealthCheckException("Backup not run today")
 
         # If we manage to get here we know that the task was run recently and

api/document_data/tests/test_health_checks.py

@@ -1,4 +1,5 @@
 import celery
+import datetime
 import uuid
 
 from unittest.mock import patch
@@ -262,3 +263,44 @@ def test_backup_document_data_files_created_during_backup_ignored(self, mock_app
         ].remaining_estimate
         mock_remaining_estimate.return_value = timezone.timedelta(hours=16)
         self.assertIsNone(self.backend.check_status())
+
+    @parameterized.expand(
+        [
+            ("2024-05-22 01:49:00", -1),
+            ("2024-05-22 02:55:00", -1),
+            ("2024-05-22 14:35:00", -1),
+        ]
+    )
+    def test_bst_handled_correctly(self, check_status_time, tz_offset):
+        task_id = uuid.uuid4()
+
+        # We run the cron task at 2am but celery/django correctly handles our
+        # timezone changes so that this shifts to 1am in UTC when we are in BST
+        # We need to make sure that the healthcheck takes this into account
+        started_at = timezone.datetime(
+            2024,
+            5,
+            22,
+            1,
+            0,
+            0,
+            tzinfo=timezone.timezone.utc,
+        )
+        ended_at = timezone.datetime(
+            2024,
+            5,
+            22,
+            1,
+            50,
+            52,
+            tzinfo=timezone.timezone.utc,
+        )
+        backup_log = BackupLog.objects.create(
+            ended_at=ended_at,
+            task_id=task_id,
+        )
+        backup_log.started_at = started_at
+        backup_log.save()
+
+        with freeze_time(check_status_time, tz_offset=tz_offset):
+            self.assertIsNone(self.backend.check_status())

api/external_data/documents.py

@@ -89,6 +89,7 @@ class DenialEntityDocument(Document):
 
     is_revoked = fields.BooleanField(attr="denial.is_revoked")
     notifying_government = fields.KeywordField(attr="denial.notifying_government")
+    denial_cle = fields.TextField(attr="denial.denial_cle")
     denial = fields.ObjectField(
         attr="denial",
         properties={
@@ -104,12 +105,6 @@ class DenialEntityDocument(Document):
                     "raw": fields.KeywordField(),
                 },
             ),
-            "item_list_codes": fields.TextField(
-                attr="item_list_codes",
-                fields={
-                    "raw": fields.KeywordField(),
-                },
-            ),
             "item_description": fields.TextField(
                 attr="item_description",
                 fields={

api/external_data/management/commands/ingest_denials.py

@@ -39,7 +39,7 @@ class Command(BaseCommand):
         "reference",
         "regime_reg_ref",
         "notifying_government",
-        "item_list_codes",
+        "denial_cle",
         "item_description",
         "end_use",
         "reason_for_refusal",

api/external_data/management/commands/tests/test_ingest_denials.py

@@ -28,7 +28,7 @@ def json_file_data():
                         "address": "somewhere\nmid\nlatter\nCairo",
                         "notifying_government": "United Kingdom",
                         "country": "United States",
-                        "item_list_codes": "123456",
+                        "denial_cle": "123456",
                         "item_description": "phone",
                         "end_use": "locating phone",
                         "end_user_flag": "true",
@@ -43,7 +43,7 @@ def json_file_data():
                         "address": "no address given",
                         "notifying_government": "Germany",
                         "country": "France",
-                        "item_list_codes": "12345\/2009",
+                        "denial_cle": "12345\/2009",
                         "item_description": "testing machine",
                         "end_use": "For teaching purposes",
                         "end_user_flag": "false",
@@ -98,7 +98,7 @@ def test_populate_denials(mock_json_content, mock_delete_file, json_file_data):
     assert denial_record.address == "somewhere\nmid\nlatter\nCairo"
     assert denial_record.denial.notifying_government == "United Kingdom"
     assert denial_record.country == "United States"
-    assert denial_record.denial.item_list_codes == "123456"
+    assert denial_record.denial.denial_cle == "123456"
     assert denial_record.denial.item_description == "phone"
     assert denial_record.denial.end_use == "locating phone"
     assert denial_record.denial.regime_reg_ref == "12"

api/external_data/migrations/0026_rename_item_list_codes_to_denial_cle.py

@@ -0,0 +1,15 @@
+# Generated by Django 4.2.13 on 2024-05-21 13:17
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("external_data", "0025_entity_type_empty_strings"),
+    ]
+
+    operations = [
+        migrations.RenameField("Denial", "item_list_codes", "denial_cle"),
+        migrations.RenameField("DenialEntity", "item_list_codes", "denial_cle"),
+    ]

api/external_data/models.py

@@ -18,7 +18,7 @@ class Denial(TimestampableModel):
     notifying_government = models.TextField(
         help_text="The authority that raised the denial", null=True, blank=True, default=""
     )
-    item_list_codes = models.TextField("The codes of the items being denied", null=True, blank=True, default="")
+    denial_cle = models.TextField("The codes of the items being denied", null=True, blank=True, default="")
     item_description = models.TextField("The description of the item being denied", null=True, blank=True, default="")
     end_use = models.TextField(null=True, blank=True, default="")
     is_revoked = models.BooleanField(default=False, help_text="If true do not include in search results")
@@ -48,7 +48,7 @@ class DenialEntity(TimestampableModel):
         help_text="The authority that raised the denial", blank=True, default="", null=True
     )
     country = models.TextField(blank=True, default="", null=True)
-    item_list_codes = models.TextField("The codes of the items being denied", blank=True, default="", null=True)
+    denial_cle = models.TextField("The codes of the items being denied", blank=True, default="", null=True)
     item_description = models.TextField("The description of the item being denied", blank=True, default="", null=True)
     consignee_name = models.TextField(blank=True, default="", null=True)
     end_use = models.TextField(blank=True, default="", null=True)

api/external_data/serializers.py

@@ -24,7 +24,7 @@ class Meta:
             "created_by_user",
             "regime_reg_ref",
             "notifying_government",
-            "item_list_codes",
+            "denial_cle",
             "item_description",
             "end_use",
             "is_revoked",
@@ -42,7 +42,7 @@ class Meta:
 class DenialEntitySerializer(serializers.ModelSerializer):
     entity_type = KeyValueChoiceField(choices=models.DenialEntityType.choices, required=False)
     regime_reg_ref = serializers.CharField(source="denial.regime_reg_ref", required=False)
-    item_list_codes = serializers.CharField(source="denial.item_list_codes", required=False)
+    denial_cle = serializers.CharField(source="denial.denial_cle", required=False)
     notifying_government = serializers.CharField(source="denial.notifying_government", required=False)
     item_description = serializers.CharField(source="denial.item_description", required=False)
     end_use = serializers.CharField(source="denial.end_use", required=False)
@@ -61,7 +61,7 @@ class Meta:
             "regime_reg_ref",
             "notifying_government",
             "country",
-            "item_list_codes",
+            "denial_cle",
             "item_description",
             "end_use",
             "data",
@@ -116,7 +116,7 @@ class DenialFromCSVFileSerializer(serializers.Serializer):
         "reference",
         "regime_reg_ref",
         "notifying_government",
-        "item_list_codes",
+        "denial_cle",
         "item_description",
         "end_use",
         "reason_for_refusal",
@@ -239,11 +239,11 @@ class DenialSearchSerializer(DocumentSerializer):
     entity_type = KeyValueChoiceField(choices=models.DenialEntityType.choices, required=False)
     regime_reg_ref = serializers.ReadOnlyField(source="denial.regime_reg_ref")
     reference = serializers.ReadOnlyField(source="denial.reference")
-    item_list_codes = serializers.ReadOnlyField(source="denial.item_list_codes")
     item_description = serializers.ReadOnlyField(source="denial.item_description")
     end_use = serializers.ReadOnlyField(source="denial.end_use")
     name = serializers.SerializerMethodField()
     address = serializers.SerializerMethodField()
+    denial_cle = serializers.SerializerMethodField()
 
     class Meta:
         document = documents.DenialEntityDocument
@@ -253,20 +253,25 @@ class Meta:
             "country",
             "name",
             "notifying_government",
+            "denial_cle",
         )
 
     def get_entity_type(self, obj):
         return get_denial_entity_type(obj.data.to_dict())
 
     def get_name(self, obj):
-        if hasattr(obj.meta, "highlight") and obj.meta.highlight.to_dict().get("name"):
-            return obj.meta.highlight.to_dict().get("name")[0]
-        return obj.name
+        return self.get_highlighted_field(obj, "name")
 
     def get_address(self, obj):
-        if hasattr(obj.meta, "highlight") and obj.meta.highlight.to_dict().get("address"):
-            return obj.meta.highlight.to_dict().get("address")[0]
-        return obj.address
+        return self.get_highlighted_field(obj, "address")
+
+    def get_denial_cle(self, obj):
+        return self.get_highlighted_field(obj, "denial_cle")
+
+    def get_highlighted_field(self, obj, field_name):
+        if hasattr(obj.meta, "highlight") and obj.meta.highlight.to_dict().get(field_name):
+            return obj.meta.highlight.to_dict().get(field_name)[0]
+        return getattr(obj, field_name)
 
 
 class SanctionMatchSerializer(serializers.ModelSerializer):

api/external_data/tests/denial_invalid.csv

@@ -1,4 +1,4 @@
-name,address,notifying_government,country,item_list_codes,item_description,consignee_name,end_use,end_user_flag,consignee_flag,other_role
+name,address,notifying_government,country,denial_cle,item_description,consignee_name,end_use,end_user_flag,consignee_flag,other_role
 Jim Example,123 fake street,France,Germany,ABC123,Foo,Fred Food,used in car,true,true,false
 Jak Example,123 fake street,France,Germany,ABC123,Foo,Fred Food,used in car,false,true,false
 Bob Example,123 fake street,France,Germany,ABC123,Foo,Fred Food,used in car,false,false,true

api/external_data/tests/denial_valid.csv

@@ -1,4 +1,4 @@
-reference,regime_reg_ref,name,address,notifying_government,country,item_list_codes,item_description,end_use,reason_for_refusal,entity_type
+reference,regime_reg_ref,name,address,notifying_government,country,denial_cle,item_description,end_use,reason_for_refusal,entity_type
 DN2000/0000,AB-CD-EF-000,Organisation Name,"1000 Street Name, City Name",Country Name,Country Name,0A00100,Medium Size Widget,Used in industry,Risk of outcome,end_user
 DN2000/0010,AB-CD-EF-300,Organisation Name 3,"2001 Street Name, City Name 3",Country Name 3,Country Name 3,0A00201,Unspecified Size Widget,Used in other industry,Risk of outcome 3,consignee
 ,AB-XY-EF-900,The Widget Company,"2 Example Road, Example City",Example Country,Country Name X,"catch all",Extra Large Size Widget,Used in unknown industry,Risk of outcome 4,end_user