Merge pull request #498 from geoadmin/develop

New Release v1.30.0 - #minor

.gitignore

@@ -67,6 +67,7 @@ nose2-junit.xml
 
 # ignore local files
 **/settings.py
+.env
 .env.*local
 .volumes
 **/logs

Makefile

@@ -229,16 +229,14 @@ lint-specs:
 
 
 .PHONY: ci-build-check-specs
-ci-build-check-specs:
-	@echo "Ignore ci-build-check-specs"
-	# Currently the ci-build-check-specs doesn't work because the merged output of yq
-	# differ on the ci from our ubuntu development machine.
-	# @if [[ -n `git status --porcelain --untracked-files=no` ]]; then \
-	#  	>&2 echo "ERROR: the following files changed after building the spec"; \
-	#  	>&2 echo "'git status --porcelain' reported changes in those files after a 'build-specs' :"; \
-	# 	>&2 git status --porcelain --untracked-files=no; \
-	# 	exit 1; \
-	# fi
+ci-build-check-specs: build-specs
+	@if [[ -n `git status --porcelain --untracked-files=no` ]]; then \
+		>&2 echo "ERROR: the following files changed after building the spec"; \
+		>&2 echo "'git status --porcelain' reported changes in those files after a 'build-specs' :"; \
+		>&2 git status --porcelain --untracked-files=no; \
+		>&2 git diff; \
+		exit 1; \
+	fi
 
 
 ###################

Pipfile

@@ -22,6 +22,7 @@ stac-api-validator = "~=0.6.2"
 pytest = "*"
 pytest-django = "*"
 pytest-dotenv = "*"
+pytest-cov = "*"
 
 [packages]
 gevent = "~=24.2.1"
@@ -43,9 +44,9 @@ py-multihash = "~=2.0.1"
 django-prometheus = "~=2.3.1"
 django-admin-autocomplete-filter = "~=0.7.1"
 django-pgtrigger = "~=4.11.1"
-logging-utilities = "~=4.5.0"
 django-environ = "*"
 language-tags = "*"
+logging-utilities = "~=5.0.0"
 
 [requires]
 python_version = "3.12"

README.md

@@ -430,16 +430,7 @@ With the following commands it is possible to get a proper state of the database
 
 ## Initial Setup up the RDS database and the user
 
-Right now the initial setup on the RDS database for the stagings *dev*, *int* and *prod* can be obtained
-with the helper script `scripts/setup_rds_db.sh`. The credentials come from `ssm`. To
-setup the RDS database on int, run following command (the PROFILE variable denotes in what account the
-parameters are stored):
-
-```bash
-    summon -p `ssm` -D APP_ENV=int -D PROFILE=swisstopo-bgdi-dev scripts/setup_rds_db.sh
-```
-
-**Note:** The script won't delete the existing database.
+The RDS database and user are managed by terraform. See [the service-stac module for int](https://github.com/geoadmin/infra-terraform-bgdi/blob/master/tf/aws/swisstopo-bgdi/systems-int/data/service-stac/db.tf) for example.
 
 ## Deploying the project and continuous integration
 

app/stac_api/admin.py

@@ -251,8 +251,8 @@ class Media:
                     'forecast_reference_datetime',
                     'forecast_horizon',
                     'forecast_duration',
-                    'forecast_param',
-                    'forecast_mode',
+                    'forecast_variable',
+                    'forecast_perturbed',
                 )
             }
         ),

app/stac_api/management/commands/dummy_asset_upload.py

@@ -0,0 +1,125 @@
+import logging
+import os
+
+from django.conf import settings
+from django.core.management.base import BaseCommand
+
+from stac_api.models import Asset
+from stac_api.models import AssetUpload
+from stac_api.models import BaseAssetUpload
+from stac_api.s3_multipart_upload import MultipartUpload
+from stac_api.utils import AVAILABLE_S3_BUCKETS
+from stac_api.utils import CommandHandler
+from stac_api.utils import get_asset_path
+from stac_api.utils import get_sha256_multihash
+
+logger = logging.getLogger(__name__)
+
+
+class DummyAssetUploadHandler(CommandHandler):
+
+    def __init__(self, *args, **kwargs):
+
+        # Note: the command is currently just used to be able to manipulate
+        # AssetUpload objects, not to actually upload content to a bucket, so
+        # the bucket configuration here is just to be able to create a MultipartUpload
+        # object.
+        s3_bucket = kwargs.pop('s3_bucket', AVAILABLE_S3_BUCKETS.legacy)
+        super().__init__(*args, **kwargs)
+        self.print_success(
+            f"connect MultipartUploader to s3 bucket "
+            f"{settings.AWS_SETTINGS[s3_bucket.name]['S3_BUCKET_NAME']}"
+        )
+        self.uploader = MultipartUpload(s3_bucket)
+
+    def start(self):
+        self.print_success(f"Starting upload for {self.options['asset_id']}")
+        asset = Asset.objects.filter(name=self.options['asset_id']).first()
+        if not asset:
+            self.print_error(f"asset with id {self.options['asset_id']} doesn't exist")
+            return
+
+        key = get_asset_path(asset.item, asset.name)
+        self.print_success(f" -- {key} (with pk {asset.id})")
+        size = 1000
+        file_like = os.urandom(size)
+        checksum_multihash = get_sha256_multihash(file_like)
+
+        upload_id = self.uploader.create_multipart_upload(
+            key=key,
+            asset=asset,
+            checksum_multihash=checksum_multihash,
+            update_interval=360,
+            content_encoding=None
+        )
+        AssetUpload.objects.get_or_create(
+            asset=asset, upload_id=upload_id, number_parts=1, md5_parts={"1": "ASDF"}
+        )
+        self.print_success(f" -- upload_id: {upload_id}")
+
+    def list(self):
+        for upload in AssetUpload.objects.filter(status=BaseAssetUpload.Status.IN_PROGRESS):
+            print(f"> {upload.upload_id} (asset: {upload.asset.name})")
+
+    def complete(self):
+        try:
+            upload = AssetUpload.objects.get(upload_id=self.options['upload_id'])
+            upload.status = BaseAssetUpload.Status.COMPLETED
+            upload.save()
+        except AssetUpload.DoesNotExist:
+            self.print_error(f"upload_id '{self.options['upload_id']}' doesn't exist")
+
+    def abort(self):
+        try:
+            upload = AssetUpload.objects.get(upload_id=self.options['upload_id'])
+            upload.status = BaseAssetUpload.Status.ABORTED
+            upload.save()
+        except AssetUpload.DoesNotExist:
+            self.print_error(f"upload_id {self.options['upload_id']} doesn't exist")
+
+
+class Command(BaseCommand):
+    help = """Start dummy Multipart upload for asset file on S3 for testing.
+    """
+
+    def add_arguments(self, parser):
+
+        subparsers = parser.add_subparsers(
+            dest='action',
+            required=True,
+            help='Define the action to be performed, either "start" (default) to initate '
+            ' and upload, "complete" to end it or "abort" to cancel it'
+        )
+
+        # create the parser for the "start" command
+        parser_start = subparsers.add_parser('start', help='start help')
+        parser_start.add_argument(
+            '--asset-id',
+            type=str,
+            required=True,
+            help="The asset-id for which data should be uploaded"
+        )
+        parser_start.set_defaults()
+
+        # create the parser for the "complete" command
+        parser_complete = subparsers.add_parser('complete', help='complete help')
+        parser_complete.add_argument(
+            '--upload-id', type=str, required=True, help="The upload-id to complete"
+        )
+
+        # create the parser for the "abort" command
+        parser_abort = subparsers.add_parser('abort', help='abort help')
+        parser_abort.add_argument(
+            '--upload-id', type=str, required=True, help="The upload-id to abort"
+        )
+
+    def handle(self, *args, **options):
+        handler = DummyAssetUploadHandler(self, options)
+        if options['action'] == 'start':
+            handler.start()
+        elif options['action'] == 'list':
+            handler.list()
+        elif options['action'] == 'complete':
+            handler.complete()
+        elif options['action'] == 'abort':
+            handler.abort()

app/stac_api/management/commands/remove_expired_items.py

@@ -4,6 +4,8 @@
 from django.core.management.base import CommandParser
 from django.utils import timezone
 
+from stac_api.models import AssetUpload
+from stac_api.models import BaseAssetUpload
 from stac_api.models import Item
 from stac_api.utils import CommandHandler
 from stac_api.utils import CustomBaseCommand
@@ -26,6 +28,15 @@ def run(self):
         ).all()
         for item in items:
             assets = item.assets.all()
+            uploads_in_progress = AssetUpload.objects.filter(
+                asset__in=assets, status=BaseAssetUpload.Status.IN_PROGRESS
+            )
+            if uploads_in_progress.count() > 0:
+                self.print_warning(
+                    "WARNING: There are still pending asset uploads for expired items. "
+                    "These are likely stale, so we'll abort them"
+                )
+                uploads_in_progress.update(status=BaseAssetUpload.Status.ABORTED)
             assets_length = len(assets)
             self.delete(assets, 'assets')
             self.delete(item, 'item')

app/stac_api/management/commands/update_asset_file_size.py

@@ -45,6 +45,13 @@ def update(self):
                 asset.save()
                 print(".", end="", flush=True)
             except ClientError:
+                # We set file_size to None to indicate that this asset couldn't be
+                # found on the bucket. That way the script won't get stuck with the
+                # same 100 inexistent assets on one hand and we'll be able to
+                # produce a list of missing files on the other hand
+                asset.file_size = None
+                asset.save()
+                print("_", end="", flush=True)
                 logger.error(
                     'file size could not be read from s3 bucket [%s] for asset %s', bucket, key
                 )
@@ -56,7 +63,7 @@ def update(self):
 
         self.print_success(
             f"Update file size for {len(collection_assets)} collection assets out of "
-            "{total_asset_count}"
+            f"{total_asset_count}"
         )
 
         for collection_asset in collection_assets:
@@ -71,6 +78,13 @@ def update(self):
                 collection_asset.save()
                 print(".", end="", flush=True)
             except ClientError:
+                # We set file_size to None to indicate that this asset couldn't be
+                # found on the bucket. That way the script won't get stuck with the
+                # same 100 inexistent assets on one hand and we'll be able to
+                # produce a list of missing files on the other hand
+                collection_asset.file_size = None
+                collection_asset.save()
+                print("_", end="", flush=True)
                 logger.error(
                     'file size could not be read from s3 bucket [%s] for collection asset %s'
                 )