Merge branch 'master' into TP2000-1471--task-workflow

common/middleware.py

@@ -1,4 +1,5 @@
 from django.conf import settings
+from django.middleware.security import SecurityMiddleware
 from django.shortcuts import redirect
 from django.urls import reverse
 
@@ -16,3 +17,13 @@ def __call__(self, request):
 
         response = self.get_response(request)
         return response
+
+
+class CustomSecurityMiddleware(SecurityMiddleware):
+    """Extends the security middleware to include the X-Permitted-Cross-Domain-
+    Policies HTTP response header."""
+
+    def process_response(self, request, response):
+        response = super().process_response(request, response)
+        response["X-Permitted-Cross-Domain-Policies"] = "none"
+        return response

common/static/common/scss/_quota-definitions.scss

@@ -28,4 +28,10 @@
 
 .definition-original {
     color: $govuk-secondary-text-colour;
+}
+
+.bulk-create-review {
+    details.govuk-details[open] {
+        height: 420px !important;
+    }
 }

common/templates/common/widgets/decimal_suffix.html

@@ -0,0 +1,16 @@
+<div class="govuk-input__wrapper">
+	<input 
+        class="numberinput govuk-input govuk-input govuk-input--width-2" 
+        type="{{ widget.type }}" 
+        name="{{ widget.name }}"
+        {% if widget.value != None %} 
+            value="{{ widget.value|stringformat:'s' }}"
+        {% endif %}
+        {% include "django/forms/widgets/attrs.html" %}>
+    {% if widget.suffix %}
+        <div class="govuk-input__suffix" aria-hidden="true">{{widget.suffix}}</div>
+    {% endif %}
+	<label{% if widget.attrs.id %} id="{{ widget.attrs.id }}-label" for="{{ widget.attrs.id }}"{% endif %} class="govuk-label">
+		{{ widget.label }}
+	</label>
+</div>

common/widgets.py

@@ -71,3 +71,23 @@ class MultipleFileInput(FileInput):
     """Enable multiple files to be selected using FileInput widget."""
 
     allow_multiple_selected = True
+
+
+class DecimalSuffix(widgets.NumberInput):
+    """Identical to the Decimal widget but allows the addition of a suffix."""
+
+    template_name = "common/widgets/decimal_suffix.html"
+
+    def __init__(self, attrs=None, suffix="", **kwargs):
+        self.suffix = suffix
+        super().__init__(attrs, **kwargs)
+
+    def render(self, name, value, attrs=None, renderer=None):
+        if attrs is None:
+            attrs = {}
+        context = self.get_context(name, value, attrs)
+        if self.suffix:
+            context["widget"]["suffix"] = self.suffix
+
+        template = loader.get_template(self.template_name).render(context)
+        return mark_safe(template)

measures/forms/mixins.py

@@ -247,7 +247,7 @@ def conditions_clean(
             )
 
         if price:
-            date = measure_start_date or make_aware(datetime.now())
+            date = measure_start_date or make_aware(datetime.datetime.now())
             parser = LarkDutySentenceParser(compound_duties=False, date=date)
             try:
                 components = parser.transform(price)

measures/jinja2/includes/measures/conditions.jinja

@@ -1,6 +1,6 @@
-{% macro conditions_list(measure) -%}
+{% macro conditions_list(measure, workbasket) -%}
   <ol class="govuk-list">
-    {% for condition in measure.conditions.current().with_reference_price_string() -%}
+    {% for condition in measure.conditions.approved_up_to_transaction(workbasket.transactions.last()).with_reference_price_string() -%}
       <li title="{{ condition.description }}">
 
         <span class="condition_code">

measures/jinja2/includes/measures/list.jinja

@@ -68,7 +68,7 @@
                 {"text": create_link(measure.order_number.get_url(), measure.order_number.order_number) if measure.order_number else '-'},
                 {"text": footnotes_display(measure.footnoteassociationmeasure_set.current())},
                 {"text": create_link(url("regulation-ui-detail", kwargs={"role_type": measure.generating_regulation.role_type,"regulation_id": measure.generating_regulation.regulation_id}), measure.generating_regulation.regulation_id) if measure.generating_regulation.regulation_id else '-'},
-                {"text": conditions_list(measure) if measure.conditions.current() else "-", "classes": "govuk-!-width-one-quarter"},
+                {"text": conditions_list(measure, workbasket) if measure.conditions.current() else "-", "classes": "govuk-!-width-one-quarter"},
               ]) or "" }} 
           {% endfor %}
         {{ govukTable({

measures/jinja2/includes/measures/workbasket-measures.jinja

@@ -17,8 +17,8 @@
     {"html": create_link(url("additional_code-ui-detail", kwargs={"sid": measure.additional_code.sid}), measure.additional_code.type.sid ~ measure.additional_code.code) if measure.additional_code else '-'},
     {"html": create_link(url("geo_area-ui-detail", kwargs={"sid": measure.geographical_area.sid}), measure.geographical_area.area_id ~ " - " ~ measure.geographical_area.get_description().description) if measure.geographical_area else '-'},
     {"text": create_link(measure.order_number.get_url(), measure.order_number.order_number) if measure.order_number else '-'},
-    {"text": footnotes_display(measure.footnoteassociationmeasure_set.current())},
-    {"text": conditions_list(measure) if measure.conditions.current() else "-"},
+    {"text": footnotes_display(measure.footnoteassociationmeasure_set.approved_up_to_transaction(workbasket.transactions.last()))},
+    {"text": conditions_list(measure, workbasket) if measure.conditions.approved_up_to_transaction(workbasket.transactions.last()) else "-"},
   ]) or "" }}
 {% endfor %}
 

measures/tests/conftest.py

@@ -425,7 +425,7 @@ def measure_conditions_form_data(
         },
         "kwargs": {
             "form_kwargs": {
-                "measure_start_date": datetime.date(2025, 1, 1),
+                "measure_start_date": date_ranges.normal.lower,
                 "measure_type": factories.MeasureTypeFactory.create(
                     valid_between=date_ranges.normal,
                 ),
@@ -451,7 +451,7 @@ def measure_footnotes_form_data():
 
 
 @pytest.fixture()
-def measure_commodities_and_duties_form_data():
+def measure_commodities_and_duties_form_data(date_ranges):
     commodity_1 = factories.GoodsNomenclatureFactory.create()
     commodity_2 = factories.GoodsNomenclatureFactory.create()
 
@@ -464,7 +464,7 @@ def measure_commodities_and_duties_form_data():
         },
         "kwargs": {
             "min_commodity_count": 1,
-            "measure_start_date": datetime.date(2025, 1, 1),
+            "measure_start_date": date_ranges.normal.lower,
             "form_kwargs": {
                 "measure_type": None,
             },

measures/views/search.py

@@ -241,6 +241,7 @@ def get_context_data(self, **kwargs):
                     page.paginator.num_pages,
                 ),
                 "selected_filter_lists": self.selected_filter_formatter(),
+                "workbasket": self.workbasket,
             },
         )
         if context["has_previous_page"]:

publishing/management/commands/publish_to_api.py

@@ -8,28 +8,40 @@
 
 
 class Command(BaseCommand):
-    help = "Upload unpublished envelopes to the Tariff API."
+    help = (
+        "Manage envelope uploads to Tariff API. Without arguments, this "
+        "management command lists unpublished envelopes."
+    )
 
     def add_arguments(self, parser):
         parser.add_argument(
-            "-l",
-            "--list",
-            dest="list",
+            "--publish-async",
             action="store_true",
-            help="List unpublished envelopes.",
+            help=(
+                "Asynchronously run (via a Celery task) the function to "
+                "upload unpublished envelopes to the tariffs-api service."
+            ),
+        )
+        parser.add_argument(
+            "--publish-now",
+            action="store_true",
+            help=(
+                "Immediately run (within the current terminal's process) the "
+                "function to upload unpublished envelopes to the tariffs-api "
+                "service."
+            ),
         )
 
     def get_incomplete_envelopes(self):
-        incomplete = CrownDependenciesEnvelope.objects.unpublished()
-        if not incomplete:
-            return None
-        return incomplete
+        return CrownDependenciesEnvelope.objects.unpublished()
 
     def get_unpublished_envelopes(self):
-        unpublished = PackagedWorkBasket.objects.get_unpublished_to_api()
-        if not unpublished:
-            return None
-        return unpublished
+        return PackagedWorkBasket.objects.get_unpublished_to_api()
+
+    def print_envelope_details(self, position, envelope):
+        self.stdout.write(
+            f"position={position}, pk={envelope.pk}, envelope_id={envelope.envelope_id}",
+        )
 
     def list_unpublished_envelopes(self):
         incomplete = self.get_incomplete_envelopes()
@@ -39,22 +51,33 @@ def list_unpublished_envelopes(self):
                 f"{incomplete.count()} envelope(s) not completed publishing:",
             )
             for i, crowndependencies in enumerate(incomplete, start=1):
-                self.stdout.write(
-                    f"{i}: {crowndependencies.packagedworkbaskets.last().envelope}",
+                self.print_envelope_details(
+                    position=i,
+                    envelope=crowndependencies.packagedworkbaskets.last().envelope,
                 )
         if unpublished:
             self.stdout.write(
                 f"{unpublished.count()} envelope(s) ready to be published in the following order:",
             )
             for i, packaged_work_basket in enumerate(unpublished, start=1):
-                self.stdout.write(f"{i}: {packaged_work_basket.envelope}")
-
-    def handle(self, *args, **options):
-        if options["list"]:
-            self.list_unpublished_envelopes()
-            return
+                self.print_envelope_details(
+                    position=i,
+                    envelope=packaged_work_basket.envelope,
+                )
 
+    def publish(self, now: bool):
         if self.get_unpublished_envelopes() or self.get_incomplete_envelopes():
-            publish_to_api.apply()
+            if now:
+                self.stdout.write(f"Calling `publish_to_api()` now.")
+                publish_to_api()
+            else:
+                self.stdout.write(f"Calling `publish_to_api()` asynchronously.")
+                publish_to_api.apply()
         else:
             sys.exit("No unpublished envelopes")
+
+    def handle(self, *args, **options):
+        if options["publish_async"] or options["publish_now"]:
+            self.publish(now=options["publish_now"])
+        else:
+            self.list_unpublished_envelopes()

publishing/models/packaged_workbasket.py

@@ -243,12 +243,9 @@ def get_unpublished_to_api(self) -> "PackagedWorkBasketQuerySet":
         ).order_by("envelope__envelope_id")
         return unpublished
 
-    def last_unpublished_envelope_id(self) -> "publishing_models.EnvelopeId":
-        """Join PackagedWorkBasket with Envelope and CrownDependenciesEnvelope
-        model selecting objects Where an Envelope model exists and the
-        published_to_tariffs_api field is not null Or Where a
-        CrownDependenciesEnvelope is not null Then select the max value for ther
-        envelope_id field in the Envelope instance."""
+    def last_published_envelope_id(self) -> "publishing_models.EnvelopeId":
+        """Get the envelope_id of the last Envelope successfully published to
+        the Tariffs API service."""
 
         return (
             self.select_related(
@@ -409,20 +406,29 @@ def next_expected_to_api(self) -> bool:
         (this means the envelope is the first to be published to the API)
         """
 
-        previous_id = PackagedWorkBasket.objects.last_unpublished_envelope_id()
+        previous_id = PackagedWorkBasket.objects.last_published_envelope_id()
         if self.envelope.envelope_id[2:] == settings.HMRC_PACKAGING_SEED_ENVELOPE_ID:
-            year = int(self.envelope.envelope_id[:2])
-            last_envelope = publishing_models.Envelope.objects.for_year(
-                year=year - 1,
-            ).last()
-            # uses None if first envelope (no previous ones)
-            expected_previous_id = last_envelope.envelope_id if last_envelope else None
-        else:
-            expected_previous_id = str(
-                int(self.envelope.envelope_id) - 1,
+            # NOTE:
+            # Code in this conditional block, and therefore this function,
+            # wrongly assumes a new year has passed since the last envelope was
+            # successfully published to tariffs-api.
+            # See Jira ticket TP2000-1646 for details of the issue.
+
+            current_envelope_year = int(self.envelope.envelope_id[:2])
+            last_envelope_last_year = (
+                publishing_models.Envelope.objects.last_envelope_for_year(
+                    year=current_envelope_year - 1,
+                )
+            )
+            expected_previous_id = (
+                last_envelope_last_year.envelope_id if last_envelope_last_year else None
             )
+        else:
+            expected_previous_id = str(int(self.envelope.envelope_id) - 1)
+
         if previous_id and previous_id != expected_previous_id:
             return False
+
         return True
 
     # processing_state transition management.

publishing/tests/test_publishing_commands.py

@@ -20,11 +20,11 @@ def test_publish_to_api_lists_unpublished_envelopes(
     packaged_work_baskets = PackagedWorkBasket.objects.get_unpublished_to_api()
 
     out = StringIO()
-    call_command("publish_to_api", "--list", stdout=out)
+    call_command("publish_to_api", stdout=out)
     output = out.getvalue()
 
     for packaged_work_basket in packaged_work_baskets:
-        assert str(packaged_work_basket.envelope) in output
+        assert f"envelope_id={packaged_work_basket.envelope.envelope_id}" in output
 
 
 def test_publish_to_api_lists_no_envelopes(
@@ -34,7 +34,7 @@ def test_publish_to_api_lists_no_envelopes(
     settings.ENABLE_PACKAGING_NOTIFICATIONS = False
 
     out = StringIO()
-    call_command("publish_to_api", "--list", stdout=out)
+    call_command("publish_to_api", stdout=out)
     output = out.getvalue()
 
     assert not output
@@ -46,7 +46,7 @@ def test_publish_to_api_exits_no_unpublished_envelopes():
     assert CrownDependenciesEnvelope.objects.unpublished().count() == 0
 
     with pytest.raises(SystemExit):
-        call_command("publish_to_api")
+        call_command("publish_to_api", "--publish-async")
 
 
 def test_publish_to_api_publishes_envelopes(successful_envelope_factory, settings):
@@ -57,6 +57,6 @@ def test_publish_to_api_publishes_envelopes(successful_envelope_factory, setting
 
     assert PackagedWorkBasket.objects.get_unpublished_to_api().count() == 1
 
-    call_command("publish_to_api")
+    call_command("publish_to_api", "--publish-async")
 
     assert PackagedWorkBasket.objects.get_unpublished_to_api().count() == 0

quotas/forms/base.py

@@ -251,8 +251,8 @@ def init_fields(self):
         self.fields["quota_definition"].queryset = (
             models.QuotaDefinition.objects.current()
             .as_at_today_and_beyond()
-            .filter(order_number=self.quota_order_number)
-            .order_by("-sid")
+            .filter(order_number__sid=self.quota_order_number.sid)
+            .order_by("valid_between")
         )
         self.fields["quota_definition"].label_from_instance = (
             lambda obj: f"{obj.sid} ({obj.valid_between.lower} - {obj.valid_between.upper})"