Add the nature field to the Manifest model

closes: #1751

CHANGES/1751.deprecation

@@ -0,0 +1 @@
+Deprecated the Manifest `is_flatpak` and `is_bootable` fields in favor of the new `nature` field.

CHANGES/1751.feature

@@ -0,0 +1,2 @@
+Introduced the `nature` field on the Manifests endpoint to enable easier differentiation of image
+types.

pulp_container/app/management/commands/container-handle-image-data.py

@@ -39,10 +39,14 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         manifests_updated_count = 0
 
-        manifests_v1 = Manifest.objects.filter(data__isnull=True, media_type=MEDIA_TYPE.MANIFEST_V1)
+        manifests_v1 = Manifest.objects.filter(
+            Q(media_type=MEDIA_TYPE.MANIFEST_V1), Q(data__isnull=True) | Q(nature__isnull=True)
+        )
         manifests_updated_count += self.update_manifests(manifests_v1)
 
-        manifests_v2 = Manifest.objects.filter(Q(data__isnull=True) | Q(annotations={}, labels={}))
+        manifests_v2 = Manifest.objects.filter(
+            Q(data__isnull=True) | Q(annotations={}, labels={}) | Q(nature__isnull=True)
+        )
         manifests_v2 = manifests_v2.exclude(
             media_type__in=[MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI, MEDIA_TYPE.MANIFEST_V1]
         )
@@ -68,6 +72,15 @@ def handle(self, *args, **options):
     def update_manifests(self, manifests_qs):
         manifests_updated_count = 0
         manifests_to_update = []
+        fields_to_update = [
+            "annotations",
+            "labels",
+            "is_bootable",
+            "is_flatpak",
+            "data",
+            "nature",
+        ]
+
         for manifest in manifests_qs.iterator():
             # suppress non-existing/already migrated artifacts and corrupted JSON files
             with suppress(ObjectDoesNotExist, JSONDecodeError):
@@ -76,7 +89,6 @@ def update_manifests(self, manifests_qs):
                     manifests_to_update.append(manifest)
 
             if len(manifests_to_update) > 1000:
-                fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"]
                 manifests_qs.model.objects.bulk_update(
                     manifests_to_update,
                     fields_to_update,
@@ -85,7 +97,6 @@ def update_manifests(self, manifests_qs):
                 manifests_to_update.clear()
 
         if manifests_to_update:
-            fields_to_update = ["annotations", "labels", "is_bootable", "is_flatpak", "data"]
             manifests_qs.model.objects.bulk_update(
                 manifests_to_update,
                 fields_to_update,
@@ -100,11 +111,12 @@ def init_manifest(self, manifest):
             manifest_data, raw_bytes_data = get_content_data(manifest_artifact)
             manifest.data = raw_bytes_data.decode("utf-8")
 
-            if not (manifest.annotations or manifest.labels):
+            if not (manifest.annotations or manifest.labels or manifest.nature):
                 manifest.init_metadata(manifest_data)
 
             manifest._artifacts.clear()
-
             return True
 
+        elif not manifest.nature:
+            return manifest.init_image_nature()
         return False

pulp_container/app/migrations/0042_add_manifest_nature_field.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.16 on 2024-10-21 19:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('container', '0041_add_pull_through_pull_permissions'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='manifest',
+            name='nature',
+            field=models.CharField(null=True),
+        ),
+    ]

pulp_container/app/models.py

@@ -5,6 +5,7 @@
 import re
 import tempfile
 import time
+from functools import wraps
 from logging import getLogger
 
 from django.db import models
@@ -32,7 +33,7 @@
 
 from . import downloaders
 from pulp_container.app.utils import get_content_data
-from pulp_container.constants import MEDIA_TYPE, SIGNATURE_TYPE
+from pulp_container.constants import MANIFEST_TYPE, MEDIA_TYPE, SIGNATURE_TYPE
 
 
 logger = getLogger(__name__)
@@ -72,6 +73,7 @@ class Manifest(Content):
         digest (models.TextField): The manifest digest.
         schema_version (models.IntegerField): The manifest schema version.
         media_type (models.TextField): The manifest media type.
+        nature (models.TextField): The manifest's type (flatpak, bootable, signature, etc.).
         data (models.TextField): The manifest's data in text format.
         annotations (models.JSONField): Metadata stored inside the image manifest.
         labels (models.JSONField): Metadata stored inside the image configuration.
@@ -99,12 +101,15 @@ class Manifest(Content):
     digest = models.TextField(db_index=True)
     schema_version = models.IntegerField()
     media_type = models.TextField(choices=MANIFEST_CHOICES)
+    nature = models.CharField(null=True)
     data = models.TextField(null=True)
 
     annotations = models.JSONField(default=dict)
     labels = models.JSONField(default=dict)
 
+    # DEPRECATED: this field is deprecated and will be removed in a future release.
     is_bootable = models.BooleanField(default=False)
+    # DEPRECATED: this field is deprecated and will be removed in a future release.
     is_flatpak = models.BooleanField(default=False)
 
     blobs = models.ManyToManyField(Blob, through="BlobManifest")
@@ -154,39 +159,85 @@ def init_image_nature(self):
             return self.init_manifest_nature()
 
     def init_manifest_list_nature(self):
+        updated_nature = False
+        if not self.nature:
+            self.nature = MANIFEST_TYPE.UNKNOWN
+            updated_nature = True
+
         for manifest in self.listed_manifests.all():
             # it suffices just to have a single manifest of a specific nature;
             # there is no case where the manifest is both bootable and flatpak-based
             if manifest.is_bootable:
+                self.nature = MANIFEST_TYPE.BOOTABLE
                 self.is_bootable = True
                 return True
             elif manifest.is_flatpak:
+                self.nature = MANIFEST_TYPE.FLATPAK
                 self.is_flatpak = True
                 return True
 
-        return False
+        return updated_nature
 
     def init_manifest_nature(self):
-        if self.is_bootable_image():
-            self.is_bootable = True
-            return True
-        elif self.is_flatpak_image():
-            self.is_flatpak = True
-            return True
-        else:
-            return False
+        for manifest_type, check_type_function in self.known_types().items():
+            if check_type_function():
+                self.nature = manifest_type
+                return True
+
+        return False
+
+    def known_types(self):
+        return {
+            MANIFEST_TYPE.BOOTABLE: self.is_bootable_image,
+            MANIFEST_TYPE.FLATPAK: self.is_flatpak_image,
+            MANIFEST_TYPE.HELM: self.is_helm_image,
+            MANIFEST_TYPE.SIGNATURE: self.is_cosign,
+            MANIFEST_TYPE.IMAGE: self.is_manifest_image,
+        }
 
     def is_bootable_image(self):
         if (
             self.annotations.get("containers.bootc") == "1"
             or self.labels.get("containers.bootc") == "1"
         ):
+            # DEPRECATED: is_bootable is deprecated and will be removed in a future release.
+            self.is_bootable = True
             return True
         else:
             return False
 
     def is_flatpak_image(self):
-        return True if self.labels.get("org.flatpak.ref") else False
+        if self.labels.get("org.flatpak.ref"):
+            # DEPRECATED: is_flatpak is deprecated and will be removed in a future release.
+            self.is_flatpak = True
+            return True
+        return False
+
+    def is_manifest_image(self):
+        return self.media_type in (MEDIA_TYPE.MANIFEST_OCI, MEDIA_TYPE.MANIFEST_V2)
+
+    def validate_json_field(field):
+        def decorator(func):
+            @wraps(func)
+            def wrapper(self, *args, **kwargs):
+                json_manifest = json.loads(self.data)
+                if not json_manifest.get(field, None):
+                    return False
+                return func(self, json_manifest, *args, **kwargs)
+
+            return wrapper
+
+        return decorator
+
+    @validate_json_field("layers")  # Check for 'layers' field for cosign images
+    def is_cosign(self, json_manifest):
+        return any(
+            layer.get("mediaType", None) == MEDIA_TYPE.COSIGN for layer in json_manifest["layers"]
+        )
+
+    @validate_json_field("config")  # Check for 'config' field for helm images
+    def is_helm_image(self, json_manifest):
+        return json_manifest.get("config").get("mediaType") == MEDIA_TYPE.HELM
 
     class Meta:
         default_related_name = "%(app_label)s_%(model_name)s"

pulp_container/app/registry_api.py

@@ -1238,7 +1238,7 @@ def put(self, request, path, pk=None):
             # once relations for listed manifests are established, it is
             # possible to initialize the nature of the manifest list
             if manifest.init_manifest_list_nature():
-                manifest.save(update_fields=["is_bootable", "is_flatpak"])
+                manifest.save(update_fields=["is_bootable", "is_flatpak", "nature"])
 
             found_blobs = models.Blob.objects.filter(
                 digest__in=found_manifests.values_list("blobs__digest"),

pulp_container/app/serializers.py

@@ -64,6 +64,11 @@ class ManifestSerializer(NoArtifactContentSerializer):
     digest = serializers.CharField(help_text="sha256 of the Manifest file")
     schema_version = serializers.IntegerField(help_text="Manifest schema version")
     media_type = serializers.CharField(help_text="Manifest media type of the file")
+    nature = serializers.CharField(
+        help_text="Manifest type (flatpak, bootable, signature, etc.).",
+        required=False,
+        default=None,
+    )
     listed_manifests = DetailRelatedField(
         many=True,
         help_text="Manifests that are referenced by this Manifest List",
@@ -93,15 +98,23 @@ class ManifestSerializer(NoArtifactContentSerializer):
         help_text=_("Property describing metadata stored inside the image configuration"),
     )
 
+    # DEPRECATED: this field is deprecated and will be removed in a future release.
     is_bootable = serializers.BooleanField(
         required=False,
         default=False,
-        help_text=_("A boolean determining whether users can boot from an image or not."),
+        help_text=_(
+            "A boolean determining whether users can boot from an image or not."
+            "[deprecated] check nature field instead"
+        ),
     )
+    # DEPRECATED: this field is deprecated and will be removed in a future release.
     is_flatpak = serializers.BooleanField(
         required=False,
         default=False,
-        help_text=_("A boolean determining whether the image bundles a Flatpak application"),
+        help_text=_(
+            "A boolean determining whether the image bundles a Flatpak application."
+            "[deprecated] check nature field instead"
+        ),
     )
 
     class Meta:
@@ -116,6 +129,7 @@ class Meta:
             "labels",
             "is_bootable",
             "is_flatpak",
+            "nature",
         )
         model = models.Manifest
 

pulp_container/app/tasks/builder.py

@@ -83,6 +83,7 @@ def add_image_from_directory_to_repository(path, repository, tag):
 
     with repository.new_version() as new_repo_version:
         manifest_json = json.loads(manifest_text_data)
+        manifest.init_metadata(manifest_json)
 
         config_blob = get_or_create_blob(manifest_json["config"], manifest, path)
         manifest.config_blob = config_blob

pulp_container/app/tasks/sync_stages.py

@@ -392,6 +392,7 @@ def create_manifest(self, manifest_data, raw_text_data, media_type, digest=None)
             annotations=manifest_data.get("annotations", {}),
         )
 
+        manifest.init_manifest_nature()
         manifest_dc = DeclarativeContent(content=manifest)
         return manifest_dc
 
@@ -648,4 +649,4 @@ def _post_save(self, batch):
         # it is possible to initialize the nature of the corresponding manifest lists
         for ml in manifest_lists:
             if ml.init_manifest_list_nature():
-                ml.save(update_fields=["is_bootable", "is_flatpak"])
+                ml.save(update_fields=["is_bootable", "is_flatpak", "nature"])

pulp_container/constants.py

@@ -19,6 +19,8 @@
     FOREIGN_BLOB_OCI_TAR_GZIP="application/vnd.oci.image.layer.nondistributable.v1.tar+gzip",
     FOREIGN_BLOB_OCI_TAR_ZSTD="application/vnd.oci.image.layer.nondistributable.v1.tar+zstd",
     OCI_EMPTY_JSON="application/vnd.oci.empty.v1+json",
+    HELM="application/vnd.cncf.helm.config.v1+json",
+    COSIGN="application/vnd.dev.cosign.simplesigning.v1+json",
 )
 
 V2_ACCEPT_HEADERS = {
@@ -71,3 +73,12 @@
 SIGNATURE_PAYLOAD_MAX_SIZE = 4 * MEGABYTE
 
 SIGNATURE_API_EXTENSION_VERSION = 2
+
+MANIFEST_TYPE = SimpleNamespace(
+    IMAGE="image",
+    BOOTABLE="bootable",
+    FLATPAK="flatpak",
+    HELM="helm",
+    SIGNATURE="signature",
+    UNKNOWN="unknown",
+)

pulp_container/tests/functional/api/test_build_images.py

@@ -7,6 +7,7 @@
 from pulp_smash.pulp3.bindings import monitor_task
 
 from pulpcore.client.pulp_container import ApiException, ContainerContainerDistribution
+from pulp_container.constants import MANIFEST_TYPE
 
 
 @pytest.fixture
@@ -62,6 +63,7 @@ def _build_image(repository, containerfile=None, containerfile_name=None, build_
 
 def test_build_image_with_uploaded_containerfile(
     build_image,
+    check_manifest_fields,
     containerfile_name,
     container_distribution_api,
     container_repo,
@@ -85,6 +87,9 @@ def test_build_image_with_uploaded_containerfile(
     local_registry.pull(distribution.base_path)
     image = local_registry.inspect(distribution.base_path)
     assert image[0]["Config"]["Cmd"] == ["cat", "/tmp/inside-image.txt"]
+    assert check_manifest_fields(
+        manifest_filters={"digest": image[0]["Digest"]}, fields={"nature": MANIFEST_TYPE.IMAGE}
+    )
 
 
 def test_build_image_from_repo_version_with_anon_user(

pulp_container/tests/functional/api/test_pull_through_cache.py

@@ -7,6 +7,7 @@
 from subprocess import CalledProcessError
 from uuid import uuid4
 
+from pulp_container.constants import MANIFEST_TYPE
 from pulp_container.tests.functional.constants import (
     REGISTRY_V2,
     PULP_HELLO_WORLD_REPO,
@@ -38,6 +39,7 @@ def _add_pull_through_entities_to_cleanup(path):
 def pull_and_verify(
     anonymous_user,
     add_pull_through_entities_to_cleanup,
+    check_manifest_fields,
     container_pull_through_distribution_api,
     container_distribution_api,
     container_repository_api,
@@ -59,6 +61,12 @@ def _pull_and_verify(images, pull_through_distribution):
             local_registry.pull(local_image_path)
             local_image = local_registry.inspect(local_image_path)
 
+            # 1.1. check pulp manifest model fields
+            assert check_manifest_fields(
+                manifest_filters={"digest": local_image[0]["Digest"]},
+                fields={"nature": MANIFEST_TYPE.IMAGE},
+            )
+
             path, tag = local_image_path.split(":")
             tags_to_verify.append(tag)