Merge pull request #1938 from uktrade/uat

PROD Release 22_04_23

Pipfile

@@ -20,7 +20,7 @@ ipdb = "*"
 watchdog = {extras = ["watchmedo"], version = "*"}
 diff-pdf-visually = "~=1.7.0"
 pytest-circleci-parallelized = "~=0.1.0"
-moto = {extras = ["s3"], version = "*"}
+moto = {extras = ["s3"], version = "==5.0.3"}
 
 [packages]
 factory-boy = "~=2.12.0"
@@ -32,7 +32,7 @@ django-allow-cidr = "~=0.5.0"
 django-elasticsearch-dsl = "~=7.2.2"
 django-elasticsearch-dsl-drf = "~=0.22.5"
 django-environ = "~=0.9.0"
-django-health-check = "~=3.16.5"
+django-health-check = "~=3.18.1"
 django-model-utils = "~=4.3.1"
 django-sortedm2m = "~=3.1.1"
 django-staff-sso-client = "~=4.2.1"
@@ -53,7 +53,7 @@ pypdf2 = "~=1.27.5"
 cryptography = "~=42.0.0"
 sentry-sdk = "~=1.17.0"
 elastic-apm = "~=6.7.2"
-gunicorn = "~=21.2.0"
+gunicorn = "~=22.0.0"
 gevent = "~=23.9.1"
 xmltodict = "~=0.12.0"
 Pillow = "~=10.2.0"
@@ -66,12 +66,12 @@ django-extensions = "~=3.2.3"
 ipython = "~=7.34.0"
 celery = "~=5.3.0"
 redis = "~=4.4.4"
-psycopg2-binary = "~=2.9.3"
 django-test-migrations = "~=1.2.0"
 django-silk = "~=5.0.3"
 django = "~=4.2.10"
 django-queryable-properties = "~=1.9.1"
 database-sanitizer = ">=1.1.0"
+psycopg = "~=3.1.18"
 
 [requires]
 python_version = "3.8"

README.md

@@ -10,6 +10,7 @@ Service for handling backend calls in LITE.
 - Clone the repository:
   - `git clone https://github.com/uktrade/lite-api.git`
   - `cd lite-api`
+  - Install [Homebrew](https://brew.sh/)
 
 ### A note on running the service without Docker
 
@@ -56,7 +57,7 @@ Service for handling backend calls in LITE.
     - Run the `make doc-migrate` command which does all of the above in one
   - Option 3:
     - Run `docker-compose up` to start the API's Django server
-    - Run the `make first-run` command which will run the migrations, seedall and populate the databse with test data
+    - Run the `make first-run` command which will run the migrations, seedall and populate the database with test data
 
 - Starting the service for the first time
   - `docker-compose up` - to start the API's Django server
@@ -116,13 +117,13 @@ We currently use celery for async tasks and scheduling in LITE;
 
 To produce PDF documents you will also need to install WeasyPrint. Do this after installing the python packages in the Pipfile;
 
-> MacOS: https://weasyprint.readthedocs.io/en/stable/install.html#macos
+> MacOS: `brew install weasyprint`
 
 > Linux: https://weasyprint.readthedocs.io/en/stable/install.html#debian-ubuntu
 
 ## Installing endesive for document signing
 
-To digitally sign documents `endesive` requires the OS library `swig` to be installed. To install run `sudo apt-get install swig`
+To digitally sign documents `endesive` requires the OS library `swig` to be installed. To install run `brew install swig`
 
 A `.p12` file is also required. Please see https://uktrade.atlassian.net/wiki/spaces/ILT/pages/1390870733/PDF+Document+Signing
 
@@ -139,6 +140,10 @@ A `.p12` file is also required. Please see https://uktrade.atlassian.net/wiki/sp
 ER diagrams can be viewed in docs/entity-relation-diagrams/.
 
 You'll need to install any dev [graphviz](https://graphviz.org/) dependencies (on ubuntu `sudo apt install libgraphviz-dev`) and then `pygraphviz`.
+  ```
+  brew install graphviz
+  pip install pygraphviz
+  ```
 
 Gegenerate diagrams
 

api/cases/migrations/0064_update_teams_on_advice.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.11 on 2024-04-11 14:51
+
+from django.db import migrations
+
+
+def update_teams_on_advice(apps, schema_editor):
+    Advice = apps.get_model("cases", "Advice")
+
+    teams = set(Advice.objects.values_list("user__team", flat=True))
+    for team in teams:
+        advice_without_a_team = Advice.objects.filter(team__isnull=True, user__team=team)
+        advice_without_a_team.update(team=team)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("cases", "0063_ecjuquery_chaser_email_sent_on"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            update_teams_on_advice,
+            reverse_code=migrations.RunPython.noop,
+        ),
+    ]

api/cases/tests/test_migrations.py

@@ -0,0 +1,78 @@
+from django_test_migrations.contrib.unittest_case import MigratorTestCase
+
+from api.core.constants import Roles
+
+
+class TestAdviceTeamMigration(MigratorTestCase):
+    migrate_from = ("cases", "0063_ecjuquery_chaser_email_sent_on")
+    migrate_to = ("cases", "0064_update_teams_on_advice")
+
+    def prepare(self):
+        Role = self.old_state.apps.get_model("users", "Role")
+        Role.objects.create(id=Roles.INTERNAL_DEFAULT_ROLE_ID)
+
+        Team = self.old_state.apps.get_model("teams", "Team")
+        self.a_team = Team.objects.create(name="a_team")
+        self.b_team = Team.objects.create(name="b_team")
+
+        BaseUser = self.old_state.apps.get_model("users", "BaseUser")
+        a_team_base_user = BaseUser.objects.create(email="[email protected]")  # /PS-IGNORE
+        b_team_base_user = BaseUser.objects.create(email="[email protected]")  # /PS-IGNORE
+
+        GovUser = self.old_state.apps.get_model("users", "GovUser")
+        a_team_user = GovUser.objects.create(
+            baseuser_ptr=a_team_base_user,
+            team=self.a_team,
+        )
+        b_team_user = GovUser.objects.create(
+            baseuser_ptr=b_team_base_user,
+            team=self.b_team,
+        )
+
+        Organisation = self.old_state.apps.get_model("organisations", "Organisation")
+        organisation = Organisation.objects.create(name="test")
+
+        CaseType = self.old_state.apps.get_model("cases", "CaseType")
+        case_type = CaseType.objects.get(pk="00000000-0000-0000-0000-000000000004")
+
+        Case = self.old_state.apps.get_model("cases", "Case")
+        case = Case.objects.create(
+            case_type=case_type,
+            organisation=organisation,
+        )
+
+        Advice = self.old_state.apps.get_model("cases", "Advice")
+        self.user_advice_with_same_team = Advice.objects.create(
+            case=case,
+            team=self.a_team,
+            user=a_team_user,
+        )
+        self.user_advice_with_different_team = Advice.objects.create(
+            case=case,
+            team=self.a_team,
+            user=b_team_user,
+        )
+        self.user_advice_without_team = Advice.objects.create(
+            case=case,
+            team=None,
+            user=a_team_user,
+        )
+
+    def test_migration_0064_update_teams_on_advice(self):
+        self.user_advice_with_same_team.refresh_from_db()
+        self.assertEqual(
+            self.user_advice_with_same_team.team,
+            self.a_team,
+        )
+
+        self.user_advice_with_different_team.refresh_from_db()
+        self.assertEqual(
+            self.user_advice_with_different_team.team,
+            self.a_team,
+        )
+
+        self.user_advice_without_team.refresh_from_db()
+        self.assertEqual(
+            self.user_advice_without_team.team,
+            self.a_team,
+        )

api/document_data/tests/test_celery_tasks.py

@@ -45,7 +45,7 @@ def test_backup_new_document_data(self):
             "thisisakey",
         )
         self.assertEqual(
-            document_data.data.tobytes(),
+            document_data.data,
             b"test",
         )
         s3_object = self.get_object_from_default_bucket("thisisakey")
@@ -92,7 +92,7 @@ def test_update_existing_document_data(self):
             "thisisakey",
         )
         self.assertEqual(
-            document_data.data.tobytes(),
+            document_data.data,
             b"new contents",
         )
         s3_object = self.get_object_from_default_bucket("thisisakey")
@@ -139,7 +139,7 @@ def test_leave_existing_document_data(self):
             "thisisakey",
         )
         self.assertEqual(
-            document_data.data.tobytes(),
+            document_data.data,
             b"test",
         )
         self.assertEqual(

api/external_data/enums.py

@@ -6,3 +6,11 @@ class DenialMatchCategory:
         (PARTIAL, "Partial"),
         (EXACT, "Exact"),
     ]
+
+
+class DenialEntityType:
+    CONSIGNEE = "consignee"
+    END_USER = "end_user"
+    THIRD_PARTY = "third_party"
+
+    choices = ((CONSIGNEE, "Consignee"), (END_USER, "End-user"), (THIRD_PARTY, "Third party"))

api/external_data/management/commands/ingest_denials.py

@@ -5,13 +5,13 @@
 from django.core.management.base import BaseCommand
 from django.db import transaction
 from api.applications.models import DenialMatchOnApplication
+from api.external_data.serializers import DenialEntitySerializer
 from rest_framework import serializers
 
 from elasticsearch_dsl import connections
 
 from api.documents.libraries import s3_operations
 from api.external_data import documents
-from api.external_data.serializers import DenialEntitySerializer
 
 from api.external_data.models import DenialEntity
 
@@ -63,15 +63,22 @@ def handle(self, *args, **options):
 
     @transaction.atomic
     def load_denials(self, filename):
-
         data = get_json_content_and_delete(filename)
+        errors = []
         if data:
             # Lets delete all denial records except ones that have been matched
             matched_denial_ids = DenialMatchOnApplication.objects.all().values_list("denial_id", flat=True).distinct()
             DenialEntity.objects.all().exclude(id__in=matched_denial_ids).delete()
 
-        errors = []
         for i, row in enumerate(data, start=1):
+            # This is required so we don't reload the same denial entity and load duplicates
+            has_fields = bool(row.get("regime_reg_ref") and row.get("name"))
+            if has_fields:
+                exists = DenialMatchOnApplication.objects.filter(
+                    denial__regime_reg_ref=row["regime_reg_ref"], denial__name=row["name"]
+                ).exists()
+                if exists:
+                    continue
             serializer = DenialEntitySerializer(
                 data={
                     "data": row,

api/external_data/management/commands/tests/test_ingest_denials.py

@@ -127,12 +127,13 @@ def test_populate_denials_validation_call(mock_json_content, mock_delete_file):
 def test_populate_denials_with_existing_matching_records(mock_get_file, mock_delete_file, json_file_data):
     mock_get_file.return_value = json_file_data
     case = StandardApplicationFactory()
-    denial = DenialMatchFactory()
-    DenialMatchOnApplicationFactory(application=case, category="exact", denial=denial)
+
+    denial_enity = DenialMatchFactory(regime_reg_ref="12", name="Test1 case")
+    DenialMatchOnApplicationFactory(application=case, category="exact", denial=denial_enity)
 
     call_command("ingest_denials", "json_file")
 
-    assert DenialEntity.objects.all().count() == 4
+    assert DenialEntity.objects.all().count() == 3
 
 
 @pytest.mark.django_db

api/external_data/migrations/0022_denialentity_entity_type.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.11 on 2024-04-18 08:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("external_data", "0021_denialentity_denial_id"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="denialentity",
+            name="entity_type",
+            field=models.TextField(
+                blank=True,
+                choices=[("consignee", "Consignee"), ("end_user", "End-user"), ("third_party", "Third party")],
+                default="",
+                help_text="Type of entity being denied",
+                null=True,
+            ),
+        ),
+    ]

api/external_data/models.py

@@ -5,6 +5,7 @@
 from api.common.models import TimestampableModel
 from api.flags.models import Flag
 from api.users.models import GovUser
+from api.external_data.enums import DenialEntityType
 
 
 class Denial(TimestampableModel):
@@ -59,6 +60,9 @@ class DenialEntity(TimestampableModel):
         help_text="Reason why the denial was refused", blank=True, default="", null=True
     )
     spire_entity_id = models.IntegerField(help_text="Entity_id from spire for matching data", null=True)
+    entity_type = models.TextField(
+        choices=DenialEntityType.choices, help_text="Type of entity being denied", blank=True, default="", null=True
+    )
 
 
 class SanctionMatch(TimestampableModel):

api/letter_templates/context_generator.py

@@ -885,11 +885,14 @@ def _get_goods_context(application, final_advice, licence=None):
     # Ensure that for each proviso final advice record, we add a record to the goods
     #  context
     for advice in final_advice:
-        if advice.good_id in good_ids_to_goods_on_application:
-            # Grab the next GoodOnApplication for this Good.id - this ensures that
-            #  each GoodOnApplication is present once on the end licence
-            good_on_application = good_ids_to_goods_on_application[advice.good_id].pop(0)
-            goods_context[advice.type].append(_get_good_on_application_context_with_advice(good_on_application, advice))
+        # Ignore final advice records where we have no associated good on application
+        # - either our mapping value is missing or is an empty list so skip it.
+        if not good_ids_to_goods_on_application.get(advice.good_id):
+            continue
+        # Grab the next GoodOnApplication for this Good.id - this ensures that
+        #  each GoodOnApplication is present once on the end licence
+        good_on_application = good_ids_to_goods_on_application[advice.good_id].pop(0)
+        goods_context[advice.type].append(_get_good_on_application_context_with_advice(good_on_application, advice))
 
     # Because we append goods that are approved with proviso to the approved goods below
     # we need to make sure only to keep approved goods that are not in proviso goods

api/letter_templates/tests/test_context_generation.py

@@ -492,6 +492,43 @@ def test_generate_context_with_advice_on_goods_missing(self):
         self.assertEqual(context["case_officer_name"], case.get_case_officer_name())
         self._assert_good_with_advice(context["goods"], final_advice, case.goods.all()[0])
 
+    def test_generate_context_with_advice_on_goods_some_missing(self):
+        """
+        This tests the scenario where advice had been given on for two
+        GoodOnApplications referring to the same Good.  After application editing,
+        only one GoodOnApplication remains despite both advice records still being present.
+        """
+        case = self.create_standard_application_case(self.organisation, user=self.exporter_user)
+        good = case.goods.first().good
+        another_good_on_application = GoodOnApplicationFactory(
+            application=case,
+            good=good,
+            quantity=10,
+            unit=Units.NAR,
+            value=500,
+        )
+        final_advice = FinalAdviceFactory(
+            user=self.gov_user,
+            case=case,
+            type=AdviceType.REFUSE,
+            good=good,
+        )
+        FinalAdviceFactory(
+            user=self.gov_user,
+            case=case,
+            type=AdviceType.REFUSE,
+            good=good,
+        )
+
+        case.goods.first().delete()  # Remove the first good from the application
+
+        context = get_document_context(case)
+        render_to_string(template_name="letter_templates/case_context_test.html", context=context)
+
+        self.assertEqual(context["case_reference"], case.reference_code)
+        self.assertEqual(context["case_officer_name"], case.get_case_officer_name())
+        self._assert_good_with_advice(context["goods"], final_advice, case.goods.all()[0])
+
     def test_generate_context_with_proviso_advice_on_goods(self):
         case = self.create_standard_application_case(self.organisation, user=self.exporter_user)
         good = case.goods.first()