TP2000-1493 - TAP driven quota open data export

.gitignore

@@ -167,3 +167,4 @@ _dumped_cache.pkl
 # Database dumps
 *.sql
 /.vscode/settings.json
+/quotas_export/**

exporter/management/commands/export_quotas.py

@@ -0,0 +1,50 @@
+import logging
+from typing import Any
+from typing import Optional
+
+from django.core.management import BaseCommand
+from django.core.management.base import CommandParser
+
+from exporter.quotas.tasks import export_and_upload_quotas_csv
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = (
+        "Create a CSV of quotas for use within data workspace to produce the "
+        "HMRC tariff open data CSV. The filename take the form "
+        "quotas_export_<yyyymmdd>.csv. Care should be taken to ensure that "
+        "there is sufficient local file system storage to accommodate the "
+        "CSV file (although it should not be very large, less than 5MB "
+        "(1.8MB at time of creation) - if you choose to target remote S3 "
+        "storage, then a temporary local copy of the file will be created "
+        "and cleaned up."
+    )
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        parser.add_argument(
+            "--asynchronous",
+            action="store_const",
+            help="Queue the CSV export task to run in an asynchronous process.",
+            const=True,
+            default=False,
+        )
+        parser.add_argument(
+            "--save-local",
+            help=(
+                "Save the quotas CSV to the local file system under the "
+                "(existing) directory given by DIRECTORY_PATH."
+            ),
+            dest="DIRECTORY_PATH",
+        )
+        return super().add_arguments(parser)
+
+    def handle(self, *args: Any, **options: Any) -> Optional[str]:
+        logger.info(f"Triggering quotas export to CSV")
+
+        local_path = options["DIRECTORY_PATH"]
+        if options["asynchronous"]:
+            export_and_upload_quotas_csv.delay(local_path)
+        else:
+            export_and_upload_quotas_csv(local_path)

exporter/quotas/__init__.py

@@ -0,0 +1,43 @@
+"""
+quotas Export
+=============
+
+The quotas export system will query the TAP database for published quota data and store in a CSV
+file.
+
+The general process is:
+
+1. query the TAP database for the correct dataset to export.
+2. Iterate the query result and create the data for the output.
+3. Write the data to the CSV file
+4. Upload the result to the designated storage (S3 or Local)
+
+
+This process has been chosen to optimise for:
+
+- Speed, query and data production speed will be a lot faster when processed at source.
+- Testability, We have the facility to test the output and process within TAP effectively
+- Adaptability, With test coverage highlighting any issues caused by database changes etc., the adaptability
+if this implementation is high
+- Data Quality, Using TAP to produce the data will improve the quality of the output as it's using the same filters
+and joins as TAP its self does - removing the need to run queries in SQL which has been problematic, and is
+difficult to maintain.
+"""
+
+import os
+import shutil
+from itertools import chain
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+import apsw
+from django.apps import apps
+from django.conf import settings
+
+from exporter.quotas import runner
+from exporter.quotas import tasks
+
+
+def make_export(quotas_csv_named_temp_file: NamedTemporaryFile):
+    quota_csv_exporter = runner.QuotaExport(quotas_csv_named_temp_file)
+    quota_csv_exporter.run()