TERRAPI-27: Assets Filtering for geosquare publication

src/Stars.Data.Tests/Harvesters/MetadataExtractorsTests.cs

@@ -43,7 +43,7 @@ public async void TestExtractors(string key, string datadir, MetadataExtraction
         {
             StacRouter stacRouter = ServiceProvider.GetService<StacRouter>();
             TestItem testNode = new TestItem(datadir);
-            IResource route = new ContainerNode(testNode, new Dictionary<string, IAsset>(testNode.Assets), "");
+            IResource route = new ItemContainerNode(testNode, new Dictionary<string, IAsset>(testNode.Assets), "");
             IDestination destination = LocalFileDestination.Create(fileSystem.Directory.CreateDirectory("out/"), route);
             destination.PrepareDestination();
 

src/Stars.Data.Tests/Translators/StacCollectionToAtomItemTests.cs

@@ -11,6 +11,8 @@
 using Terradue.ServiceModel.Ogc.Owc.AtomEncoding;
 using System;
 using System.IO;
+using Terradue.Stars.Services.Supplier;
+using Terradue.Stars.Services;
 
 namespace Terradue.Data.Tests.Translators
 {
@@ -53,6 +55,52 @@ public async System.Threading.Tasks.Task EGMS()
             Assert.True(wmsIsPresent);           
         }
 
+        [Fact]
+        public async System.Threading.Tasks.Task EGMS_2018_2022()
+        {
+            string json = GetJson("Translators");
+
+            ValidateJson(json);
+
+            StacCollection stacCollection = StacConvert.Deserialize<StacCollection>(json);
+
+            StacCollectionToAtomItemTranslator stacCollectionToAtomItemTranslator = new StacCollectionToAtomItemTranslator(ServiceProvider);
+
+            StacCollectionNode stacCollectionNode = new StacCollectionNode(stacCollection, new System.Uri("https://api.terradue.com/timeseries/v1/ns/gep-egms/cs/EGMS-2018-2022"));
+
+            // Filter assets to remove the timeseries assets
+            AssetFilters assetFilters = AssetFilters.CreateAssetFilters(
+                new string[] { "{type}!application/csv" }
+            );
+            // Create a filtered asset container
+            FilteredAssetContainer filteredAssetContainer = new FilteredAssetContainer(stacCollectionNode, assetFilters);
+            // Create a container node with the filtered asset container
+            CollectionContainerNode filteredNode = new CollectionContainerNode(stacCollectionNode, filteredAssetContainer.Assets, "filtered");
+            StacCollection stacCollection1 = new StacCollection(stacCollection);
+            stacCollection1.Assets.Clear();
+            stacCollection1.Assets.AddRange(filteredAssetContainer.Assets.ToDictionary(asset => asset.Key, asset => (asset.Value as StacAssetAsset).StacAsset));
+            StacCollectionNode stacCollectionNode1 = new StacCollectionNode(stacCollection1, new System.Uri("https://api.terradue.com/timeseries/v1/ns/gep-egms/cs/EGMS-2018-2022"));
+
+            AtomItemNode atomItemNode = await stacCollectionToAtomItemTranslator.TranslateAsync<AtomItemNode>(stacCollectionNode1, CancellationToken.None);
+
+            bool egmsIsPresent = false;
+            if (atomItemNode.AtomItem.ElementExtensions != null && atomItemNode.AtomItem.ElementExtensions.Count > 0)
+			{
+                var offerings = atomItemNode.AtomItem.ElementExtensions.ReadElementExtensions<OwcOffering>("offering", OwcNamespaces.Owc, new System.Xml.Serialization.XmlSerializer(typeof(OwcOffering)));
+
+				foreach (var offering in offerings)
+				{
+                    if(offering != null && offering.Code == "http://www.terradue.com/egms") egmsIsPresent = true;
+                }
+            }
+
+            Assert.True(egmsIsPresent);   
+
+            // Check that there is no link with the relationship type "enclosure"
+            Assert.DoesNotContain(atomItemNode.AtomItem.Links,
+                                  l => l.MediaType == "application/csv" && l.RelationshipType == "enclosure");
+        }
+
     }
 
 }

src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs

@@ -99,7 +99,7 @@ protected override async Task<StacNode> ExtractMetadata(IItem item, string suffi
             {
                 var mergedAssets = new Dictionary<string, IAsset>(item.Assets.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
                 mergedAssets.MergeAssets(extractedAssets.Assets, false);
-                innerStacItem = new ContainerNode(item, mergedAssets, "merged");
+                innerStacItem = new ItemContainerNode(item, mergedAssets, "merged");
             }
 
             IAsset metadataAsset = GetMetadataAsset(innerStacItem);

src/Stars.Data/ThirdParty/Publication/GeosquarePublicationModel.cs

@@ -38,6 +38,7 @@ public GeosquarePublicationModel(IPublicationModel pubModel)
             Collection = pubModel.Collection;
             CatalogId = pubModel.CatalogId;
             Depth = pubModel.Depth;
+            AssetsFilters = pubModel.AssetsFilters;
         }
 
         public GeosquarePublicationModel(GeosquarePublicationModel publishCatalogModel)
@@ -51,6 +52,7 @@ public GeosquarePublicationModel(GeosquarePublicationModel publishCatalogModel)
             CustomLinkUpdater = publishCatalogModel.CustomLinkUpdater;
             CatalogId = publishCatalogModel.CatalogId;
             Depth = publishCatalogModel.Depth;
+            AssetsFilters = publishCatalogModel.AssetsFilters;
         }
 
         /// <summary>
@@ -111,6 +113,8 @@ internal void UpdateLink(SyndicationLink link, AtomItem item, IAssetsContainer a
         public bool ThrowPublicationException { get; set; } = true;
 
         public string CatalogId { get; set; }
+
+        public List<string> AssetsFilters { get; set; }
     }
 
 }

src/Stars.Data/ThirdParty/Publication/GeosquareService.cs

@@ -25,6 +25,7 @@
 using System.Collections.Specialized;
 using System.Web;
 using Terradue.ServiceModel.Ogc.Owc.AtomEncoding;
+using Terradue.Stars.Services.Supplier;
 
 namespace Terradue.Stars.Data.ThirdParty.Geosquare
 {
@@ -100,7 +101,8 @@ private GeosquarePublicationModel CreateModelFromPublication(IPublicationModel p
                 AdditionalLinks = publicationModel.AdditionalLinks,
                 CreateIndex = true,
                 SubjectsList = publicationModel.Subjects?.Select(s => new Subject(s)).ToList(),
-                CatalogId = publicationModel.CatalogId ?? geosquareConfiguration.BaseUri.ToString()
+                CatalogId = publicationModel.CatalogId ?? geosquareConfiguration.BaseUri.ToString(),
+                AssetsFilters = publicationModel.AssetsFilters
             };
         }
 
@@ -148,9 +150,26 @@ public async Task<object> PostCollectionToCatalog(Terradue.Stars.Interface.IColl
         {
             GeosquarePublicationState catalogPublicationState = state as GeosquarePublicationState;
             AtomItemNode atomItemNode = null;
+
+            // Filter assets
+            // Create the asset filters based on the asset filters string from the catalog publication model
+            AssetFilters assetFilters = AssetFilters.CreateAssetFilters(catalogPublicationState.GeosquarePublicationModel.AssetsFilters);
+            // Create a filtered asset container
+            FilteredAssetContainer filteredAssetContainer = new FilteredAssetContainer(collectionNode, assetFilters);
+            // Create a container node with the filtered asset container
+            ICollection filteredNode = new CollectionContainerNode(collectionNode, filteredAssetContainer.Assets, "filtered");
+            // If the item is StacCollection, we recreate the StacCollection with the filtered assets
+            if (collectionNode is StacCollectionNode stacCollectionNode)
+            {
+                StacCollection stacCollection = new StacCollection(stacCollectionNode.StacCollection);
+                stacCollection.Assets.Clear();
+                stacCollection.Assets.AddRange(filteredAssetContainer.Assets.ToDictionary(asset => asset.Key, asset => (asset.Value as StacAssetAsset).StacAsset));
+                filteredNode = new StacCollectionNode(stacCollection, collectionNode.Uri);
+            }
+
             try
             {
-                atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(collectionNode, ct);
+                atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(filteredNode, ct);
             }
             catch (Exception e)
             {
@@ -176,9 +195,26 @@ public async Task<object> PostItemToCatalog(IItem itemNode, IRouter router, obje
         {
             GeosquarePublicationState catalogPublicationState = state as GeosquarePublicationState;
             AtomItemNode atomItemNode = null;
+
+            // Filter assets
+            // Create the asset filters based on the asset filters string from the catalog publication model
+            AssetFilters assetFilters = AssetFilters.CreateAssetFilters(catalogPublicationState.GeosquarePublicationModel.AssetsFilters);
+            // Create a filtered asset container
+            FilteredAssetContainer filteredAssetContainer = new FilteredAssetContainer(itemNode, assetFilters);
+            // Create a container node with the filtered asset container
+            IItem filteredNode = new ItemContainerNode(itemNode, filteredAssetContainer.Assets, "filtered");
+            // If the item is StacItem, we recreate the StacItem with the filtered assets
+            if (itemNode is StacItemNode stacItemNode)
+            {
+                StacItem stacItem = new StacItem(stacItemNode.StacItem);
+                stacItem.Assets.Clear();
+                stacItem.Assets.AddRange(filteredAssetContainer.Assets.ToDictionary(asset => asset.Key, asset => (asset.Value as StacAssetAsset).StacAsset));
+                filteredNode = new StacItemNode(stacItem, itemNode.Uri);
+            }
+
             try
             {
-                atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(itemNode, ct);
+                atomItemNode = await translatorManager.TranslateAsync<AtomItemNode>(filteredNode, ct);
             }
             catch (Exception e)
             {

src/Stars.Data/Translators/StacItemToAtomItemTranslator.cs

@@ -43,8 +43,10 @@ public async Task<T> TranslateAsync<T>(IResource node, CancellationToken ct) whe
 
         private StarsAtomItem CreateAtomItem(StacItemNode stacItemNode)
         {
+            StacItem item = stacItemNode.StacItem;
+
             // First, let's create our atomItem
-            StarsAtomItem atomItem = StarsAtomItem.Create(stacItemNode.StacItem, stacItemNode.Uri);
+            StarsAtomItem atomItem = StarsAtomItem.Create(item, stacItemNode.Uri);
 
             // Add EO Profile if possible
             atomItem.TryAddEarthObservationProfile(stacItemNode.StacItem);

src/Stars.Interface/ICollection.cs

@@ -6,12 +6,15 @@
 using System.Net.Mime;
 using System.Threading.Tasks;
 using Stac;
+using Stac.Collection;
 using Terradue.Stars.Interface.Router;
 
 namespace Terradue.Stars.Interface
 {
     public interface ICollection : ICatalog, IAssetsContainer
     {
-
+        StacExtent Extent { get; }
+
+        IDictionary<string, object> Properties { get; }
     }
 }

src/Stars.Interface/IPublicationModel.cs

@@ -16,5 +16,7 @@ public interface IPublicationModel
         string Collection { get; }
 
         int Depth { get; }
+
+        List<string> AssetsFilters { get; }
     }
 }

src/Stars.Services/Model/Stac/StacCollectionNode.cs

@@ -3,6 +3,7 @@
 using System.Linq;
 using System.Net;
 using Stac;
+using Stac.Collection;
 using Stars.Services.Model.Stac;
 using Terradue.Stars.Interface;
 
@@ -17,5 +18,9 @@ public StacCollectionNode(StacCollection stacCollection, Uri uri) : base(stacCol
         public IReadOnlyDictionary<string, IAsset> Assets => (stacObject as StacCollection).Assets.ToDictionary(asset => asset.Key, asset => (IAsset)new StacAssetAsset(asset.Value, this));
 
         public StacCollection StacCollection => stacObject as StacCollection;
+
+        public StacExtent Extent => StacCollection.Extent;
+
+        public IDictionary<string, object> Properties => StacCollection.Properties;
     }
 }

src/Stars.Services/Processing/ExtractArchiveAction.cs

@@ -119,7 +119,7 @@ public async Task<IResource> ProcessAsync(IResource route, IDestination destinat
 
             if (newAssets == null || newAssets.Count == 0) return route;
 
-            return new ContainerNode(route as IItem, newAssets, suffix);
+            return new ItemContainerNode(route as IItem, newAssets, suffix);
 
         }
 

src/Stars.Services/Supplier/AssetFilters.cs

@@ -24,7 +24,7 @@ public override string ToString()
             return Count == 0 ? "No filter" : string.Join(", ", this.Select(af => af.ToString()));
         }
 
-        public static AssetFilters CreateAssetFilters(string[] assetFiltersStr)
+        public static AssetFilters CreateAssetFilters(IEnumerable<string> assetFiltersStr)
         {
             AssetFilters assetFilters = new AssetFilters();
             if (assetFiltersStr == null)
@@ -153,17 +153,35 @@ public bool IsMatch(KeyValuePair<string, IAsset> asset)
     public class ContentTypeAssetFilter : IAssetFilter
     {
         private readonly string mediaType;
+
+        private readonly bool negated = false;
+
         private readonly Dictionary<string, string> parameters;
 
         public ContentTypeAssetFilter(string mediaType, Dictionary<string, string> parameters)
         {
-            this.mediaType = mediaType;
+            if( mediaType.StartsWith("!") )
+            {
+                this.mediaType = mediaType.Substring(1);
+                this.negated = true;
+            }
+            else
+            {
+                this.mediaType = mediaType;
+            }
             this.parameters = parameters;
         }
 
         public bool IsMatch(KeyValuePair<string, IAsset> asset)
         {
-            return 
+            return negated ?
+                ( string.IsNullOrEmpty(mediaType) ||
+                  !asset.Value.ContentType.MediaType.Equals(mediaType, System.StringComparison.InvariantCultureIgnoreCase)
+                ) &&
+                ( parameters == null ||
+                  parameters.Any(p => !asset.Value.ContentType.Parameters.ContainsKey(p.Key) ||
+                                      asset.Value.ContentType.Parameters[p.Key] != p.Value)
+                ) :
                 ( string.IsNullOrEmpty(mediaType) ||
                   asset.Value.ContentType.MediaType.Equals(mediaType, System.StringComparison.InvariantCultureIgnoreCase)
                 ) &&

src/Stars.Services/Supplier/CollectionContainerNode.cs

@@ -0,0 +1,63 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.ObjectModel;
+using System.IO;
+using System.Net.Mime;
+using System.Threading.Tasks;
+using GeoJSON.Net.Geometry;
+using Itenso.TimePeriod;
+using Stac.Collection;
+using Terradue.Stars.Interface;
+using Terradue.Stars.Interface.Router;
+using Terradue.Stars.Interface.Supplier.Destination;
+
+namespace Terradue.Stars.Services.Supplier
+{
+    public class CollectionContainerNode : IAssetsContainer, ICollection
+    {
+        private readonly ICollection collection;
+        private readonly IReadOnlyDictionary<string, IAsset>  assets;
+        private readonly string suffix;
+
+        public CollectionContainerNode(ICollection collection, IReadOnlyDictionary<string, IAsset> assets, string suffix)
+        {
+            this.collection = collection;
+            this.assets = assets;
+            this.suffix = suffix;
+        }
+
+        public IResource Node => collection;
+
+        public Uri Uri => collection.Uri;
+
+        public ContentType ContentType => collection.ContentType;
+
+        public ResourceType ResourceType => collection.ResourceType;
+
+        public ulong ContentLength => collection.ContentLength;
+
+        public string Title => collection.Title;
+
+        public string Id => collection.Id + suffix;
+
+        public ContentDisposition ContentDisposition => collection.ContentDisposition;
+
+        public StacExtent Geometry => collection.Extent;
+
+        public IDictionary<string, object> Properties => collection.Properties;
+
+        public IReadOnlyDictionary<string, IAsset> Assets => assets;
+
+        public StacExtent Extent => throw new NotImplementedException();
+
+        public IReadOnlyList<IResourceLink> GetLinks()
+        {
+            return collection.GetLinks();
+        }
+
+        public IReadOnlyList<IResource> GetRoutes(IRouter router)
+        {
+            return collection.GetRoutes(router);
+        }
+    }
+}

src/Stars.Services/Supplier/ContainerNode.cs → src/Stars.Services/Supplier/ItemContainerNode.cs

@@ -12,13 +12,13 @@
 
 namespace Terradue.Stars.Services.Supplier
 {
-    public class ContainerNode : IAssetsContainer, IItem
+    public class ItemContainerNode : IAssetsContainer, IItem
     {
         private readonly IItem item;
-        private readonly IDictionary<string, IAsset>  assets;
+        private readonly IReadOnlyDictionary<string, IAsset>  assets;
         private readonly string suffix;
 
-        public ContainerNode(IItem item, IDictionary<string, IAsset> assets, string suffix)
+        public ItemContainerNode(IItem item, IReadOnlyDictionary<string, IAsset> assets, string suffix)
         {
             this.item = item;
             this.assets = assets;
@@ -45,11 +45,10 @@ public ContainerNode(IItem item, IDictionary<string, IAsset> assets, string suff
 
         public IDictionary<string, object> Properties => item.Properties;
 
-        public IReadOnlyDictionary<string, IAsset> Assets => new ReadOnlyDictionary<string, IAsset>(assets);
+        public IReadOnlyDictionary<string, IAsset> Assets => assets;
 
         public ITimePeriod DateTime => item.DateTime;
 
-
         public IReadOnlyList<IResourceLink> GetLinks()
         {
             return item.GetLinks();