diff --git a/.gitignore b/.gitignore index f6c31684..65019630 100644 --- a/.gitignore +++ b/.gitignore @@ -370,3 +370,8 @@ src/global.json build/ package-lock.json +src/Stars.Data.Tests/Resources/BKA/L1C/data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR +src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/data/286806-EOL1ASARSAO1A3111864/Config +src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/data/286806-EOL1ASARSAO1A3111864/Data +src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/data/286806-EOL1ASARSAO1A3111864/Images +src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/data/286806-EOL1ASARSAO1A3111864/Quality diff --git a/.vscode/launch.json b/.vscode/launch.json index 9d60c653..cf984ae4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -852,6 +852,44 @@ // }, "enableStepFiltering": false }, + { + "name": "Copy S3 from CDS", + "type": "coreclr", + "request": "launch", + "preLaunchTask": "build", + // If you have changed target frameworks, make sure to update the program path. + "program": "${workspaceFolder}/src/Stars.Console/bin/Debug/net6.0/Stars.dll", + "args": [ + "copy", + "-v", + "-k", + "-r", + "4", + "-nohtml", + "-si", + "CDS", + "--output", + "/tmp/stars", + "https://catalogue.dataspace.copernicus.eu/stac/collections/SENTINEL-3/items/S3B_SL_1_RBT____20240207T082602_20240207T082902_20240207T092904_0179_089_221_0540_PS2_O_NR_004.SEN3" + ], + "cwd": "${workspaceFolder}/src", + // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console + "console": "internalConsole", + "stopAtEntry": false, + "logging": { + "moduleLoad": false + }, + "justMyCode": false, + "requireExactSource": false, + // "symbolOptions": { + // // "searchPaths": [ + // // "/data/workspace/Others/aws-sdk-net/sdk/src" + // // ], + // "searchMicrosoftSymbolServer": true, + // "searchNuGetOrgSymbolServer": true + // }, + "enableStepFiltering": false + }, { "name": ".NET Core Attach", "type": "coreclr", diff --git a/.vscode/settings.json b/.vscode/settings.json index 5d34b9ce..9daef346 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,6 @@ ], "omnisharp.disableMSBuildDiagnosticWarning": true, "dotnet-test-explorer.testProjectPath": "**/*Tests.@(csproj|vbproj|fsproj)", - "jira-plugin.workingProject": "DATAAUTHOR", + "jira-plugin.workingProject": "", "dotnet.defaultSolution": "src/Terradue.Stars.sln" } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 8aff9285..483aa4c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog). -## [2.18.1](https://github.com/Terradue/Stars/compare/2.18.0...2.18.1) +## [2.19.1](https://github.com/Terradue/Stars/compare/2.19.0...2.19.1) -Improved SAOCOM-1 metadata extraction +### Commits + +- Update file sizes in metadata JSON [`3fd6f4b`](https://github.com/Terradue/Stars/commit/3fd6f4b3086125af5c97083694c0a07c6137533d) +- Add S3 copy launch configuration and update S3ClientFactory [`72d9c46`](https://github.com/Terradue/Stars/commit/72d9c46e32b6e8ff65f76cf2c6c07a60e6593b2a) +- Add CONAE/SAOCOM-1 resources to .gitignore [`a347df7`](https://github.com/Terradue/Stars/commit/a347df70494a2d4b9b1ae52a37e52e7f88baf214) + +## [2.19.0](https://github.com/Terradue/Stars/compare/2.18.1...2.19.0) - 2024-02-09 + +### Merged + +- Change for nested BKA MUL/PAN bundles in nested ZIP files [`#43`](https://github.com/Terradue/Stars/pull/43) + +### Commits + +- Changes for BKA MUL/PAN combined archives [`12162b5`](https://github.com/Terradue/Stars/commit/12162b5c08629edeea7ee169662a43c9de8d6b7d) +- Initial changes for multiple inner asset containers [`bad8552`](https://github.com/Terradue/Stars/commit/bad8552fc59f63d63931ee68d4bdf879dc9bac1a) +- Ready to release 2.19.0 [`271a3e2`](https://github.com/Terradue/Stars/commit/271a3e231aabf6dcddd85335636af686d31c2a6c) + +## [2.18.1](https://github.com/Terradue/Stars/compare/2.18.0...2.18.1) - 2023-12-11 ### Merged diff --git a/package.json b/package.json index 857f0368..24776378 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,7 @@ { "dependencies": { - "auto-changelog": "^2.4.0" + "auto-changelog": "^2.4.0", + "package.json": "^2.0.1" }, "auto-changelog": { "output": "CHANGELOG.md", diff --git a/src/Stars.Console/Terradue.Stars.Console.csproj b/src/Stars.Console/Terradue.Stars.Console.csproj index 2de6dd4a..68dd3e99 100644 --- a/src/Stars.Console/Terradue.Stars.Console.csproj +++ b/src/Stars.Console/Terradue.Stars.Console.csproj @@ -2,7 +2,7 @@ Exe net6.0 - 2.19.0 + 2.19.2 Stars is a CLI for working with Spatio Temporal Catalog such as STAC but not only $(Version)-$(VersionSuffix) diff --git a/src/Stars.Data.Tests/Resources/BKA/L1C/MetadataExtractorsTests_4089411-0220-MUL-C-1-1-1139831-160712T150120-BR.json b/src/Stars.Data.Tests/Resources/BKA/L1C/MetadataExtractorsTests_4089411-0220-MUL-C-1-1-1139831-160712T150120-BR.json new file mode 100644 index 00000000..7c4bcdd6 --- /dev/null +++ b/src/Stars.Data.Tests/Resources/BKA/L1C/MetadataExtractorsTests_4089411-0220-MUL-C-1-1-1139831-160712T150120-BR.json @@ -0,0 +1,196 @@ +{ + "stac_version": "1.0.0", + "stac_extensions": [ + "https://stac-extensions.github.io/eo/v1.0.0/schema.json", + "https://stac-extensions.github.io/processing/v1.0.0/schema.json", + "https://stac-extensions.github.io/projection/v1.0.0/schema.json", + "https://stac-extensions.github.io/raster/v1.0.0/schema.json", + "https://stac-extensions.github.io/sat/v1.0.0/schema.json", + "https://stac-extensions.github.io/view/v1.0.0/schema.json" + ], + "type": "Feature", + "id": "4089411-0220-MUL-C-1-1-1139831-160712T150120-BR", + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [ + -45.93664497, + -23.94659722 + ], + [ + -45.75043403, + -23.91205729 + ], + [ + -45.83607639, + -23.52220486 + ], + [ + -46.02174913, + -23.55666016 + ], + [ + -45.93664497, + -23.94659722 + ] + ] + ] + }, + "properties": { + "datetime": "2016-07-12T15:01:20.0000000Z", + "created": "2023-02-21T13:53:27.0000000Z", + "platform": "bka", + "constellation": "bka", + "mission": "bka/kanopus-v", + "instruments": [ + "mss" + ], + "sensor_type": "optical", + "spectral_mode": "MUL", + "sat:platform_international_designator": "2012-039B", + "proj:epsg": 32723, + "proj:shape": [ + 4441, + 2633 + ], + "view:incidence_angle": 1.65, + "view:sun_azimuth": 2.59, + "view:sun_elevation": 44.38, + "processing:level": "L1C", + "title": "BKA MSS L1C 2016-07-12 15:01:20", + "providers": [ + { + "name": "NAS", + "description": "BKA (formerly known as BelKa 2) is a Belarusian remote sensing satellite developed under an agreement between the National Academy of Sciences of Belarus (NAS) and the Federal Space Agency of the Russian Federation. The BKA satellite is almost an exact copy of the Russian Kanopus-Vulkan N1 Environmental Satellite (Kanopus-V 1).", + "roles": [ + "producer", + "processor", + "licensor" + ], + "url": "https://gis.by/en/tech/bka/" + } + ], + "licence": "proprietary", + "gsd": 10.59 + }, + "bbox": [ + -46.02174913, + -23.94659722, + -45.75043403, + -23.52220486 + ], + "assets": { + "L1C_MUL": { + "type": "image/tiff; application=geotiff", + "roles": [ + "data", + "dn" + ], + "href": "data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/925_bka_1028560/PRODUCT/4089411-0220-MUL-C-1-1-1139831-160712T150120-BR/4089411-0220-MUL-C-1-1-1139831-160712T150120-BR.tif", + "title": "L1C MUL", + "filename": "4089411-0220-MUL-C-1-1-1139831-160712T150120-BR.tif", + "eo:bands": [ + { + "name": "MS1", + "common_name": "blue", + "center_wavelength": 0.492, + "full_width_half_max": 0.04 + }, + { + "name": "MS2", + "common_name": "green", + "center_wavelength": 0.558, + "full_width_half_max": 0.07 + }, + { + "name": "MS3", + "common_name": "red", + "center_wavelength": 0.675, + "full_width_half_max": 0.09 + }, + { + "name": "MS4", + "common_name": "nir", + "center_wavelength": 0.782, + "full_width_half_max": 0.1 + } + ], + "raster:bands": [ + { + "data_type": "int8", + "offset": 0.0 + }, + { + "data_type": "int8", + "offset": 0.0 + }, + { + "data_type": "int8", + "offset": 0.0 + }, + { + "data_type": "int8", + "offset": 0.0 + } + ] + }, + "metadata_MUL": { + "type": "application/xml", + "roles": [ + "metadata" + ], + "href": "data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/925_bka_1028560/PRODUCT/4089411-0220-MUL-C-1-1-1139831-160712T150120-BR/4089411-0220-MUL-C-1-1-1139831-160712T150120-BR_pasp-en.xml", + "filename": "4089411-0220-MUL-C-1-1-1139831-160712T150120-BR_pasp-en.xml" + }, + "overview_MUL": { + "type": "image/jpeg", + "roles": [ + "overview" + ], + "href": "data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/925_bka_1028560/PRODUCT/4089411-0220-MUL-C-1-1-1139831-160712T150120-BR/4089411-0220-MUL-C-1-1-1139831-160712T150120-BR_preview.jpg", + "filename": "4089411-0220-MUL-C-1-1-1139831-160712T150120-BR_preview.jpg" + }, + "L1C_PAN": { + "type": "image/tiff; application=geotiff", + "roles": [ + "data", + "dn" + ], + "href": "data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/925_bka_1028560/PRODUCT/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR.tif", + "title": "L1C PAN", + "filename": "4089411-0220-PAN-C-1-1-1139831-160712T150120-BR.tif", + "eo:bands": [ + { + "name": "PAN", + "common_name": "pan", + "center_wavelength": 0.66, + "full_width_half_max": 0.208 + } + ], + "raster:bands": [ + { + "data_type": "int8", + "offset": 0.0 + } + ] + }, + "metadata_PAN": { + "type": "application/xml", + "roles": [ + "metadata" + ], + "href": "data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/925_bka_1028560/PRODUCT/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR_pasp-en.xml", + "filename": "4089411-0220-PAN-C-1-1-1139831-160712T150120-BR_pasp-en.xml" + }, + "overview_PAN": { + "type": "image/jpeg", + "roles": [ + "overview" + ], + "href": "data/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/925_bka_1028560/PRODUCT/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR/4089411-0220-PAN-C-1-1-1139831-160712T150120-BR_preview.jpg", + "filename": "4089411-0220-PAN-C-1-1-1139831-160712T150120-BR_preview.jpg" + } + }, + "links": [] +} \ No newline at end of file diff --git a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json index 68b0bf0b..1be48bfc 100644 --- a/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json +++ b/src/Stars.Data.Tests/Resources/CONAE/SAOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json @@ -144,4 +144,4 @@ } }, "links": [] -} +} \ No newline at end of file diff --git a/src/Stars.Data/Model/Metadata/Bka/BkaMetadataExtractor.cs b/src/Stars.Data/Model/Metadata/Bka/BkaMetadataExtractor.cs index e779c142..9ad23ac0 100644 --- a/src/Stars.Data/Model/Metadata/Bka/BkaMetadataExtractor.cs +++ b/src/Stars.Data/Model/Metadata/Bka/BkaMetadataExtractor.cs @@ -80,7 +80,7 @@ protected override async Task ExtractMetadata(IItem item, string suffi var tmpDestination = LocalFileDestination.Create(_fileSystem.Directory.CreateDirectory(Path.GetDirectoryName(topZipArchiveAsset.Uri.AbsolutePath)), item, true); IAssetsContainer topZipAssets = await topZipArchiveAsset.ExtractToDestinationAsync(tmpDestination, _carrierManager, System.Threading.CancellationToken.None); - IAsset productZipAsset = GetProductZipAsset(topZipAssets); + IAsset productZipAsset = GetProductZipAsset(topZipAsset, topZipAssets); if (productZipAsset != null) { ZipArchiveAsset productZipArchiveAsset = new ZipArchiveAsset(productZipAsset, logger, resourceServiceProvider, _fileSystem); @@ -106,6 +106,11 @@ protected override async Task ExtractMetadata(IItem item, string suffi IAsset[] metadataAssets = GetMetadataAssets(item, innerZipAssetContainers); + if (metadataAssets == null) + { + throw new Exception("No metadata assets found"); + } + BkaMetadata[] metadata = await ReadMetadata(metadataAssets); StacItem stacItem = CreateStacItem(metadata); @@ -544,8 +549,13 @@ protected virtual IAsset GetTopZipAsset(IItem item) return zipAsset; } - protected virtual IAsset GetProductZipAsset(IAssetsContainer container) + protected virtual IAsset GetProductZipAsset(IAsset topAsset, IAssetsContainer container) { + if (topAsset.Uri.AbsolutePath.EndsWith("PRODUCT.zip")) + { + return topAsset; + } + IAsset zipAsset = FindFirstAssetFromFileNameRegex(container, @"PRODUCT\.zip"); return zipAsset; } diff --git a/src/Stars.Data/Terradue.Stars.Data.csproj b/src/Stars.Data/Terradue.Stars.Data.csproj index 3f0c361a..e215822d 100644 --- a/src/Stars.Data/Terradue.Stars.Data.csproj +++ b/src/Stars.Data/Terradue.Stars.Data.csproj @@ -4,7 +4,7 @@ Terradue.Stars.Data Terradue.Stars.Data Collection of data Plugins for Terradue.Stars - 2.19.0 + 2.19.2 $(Version)-$(VersionSuffix) NU1603 diff --git a/src/Stars.Services/Resources/AmazonCustomS3Config.cs b/src/Stars.Services/Resources/AmazonCustomS3Config.cs index 7435455d..c8596a59 100644 --- a/src/Stars.Services/Resources/AmazonCustomS3Config.cs +++ b/src/Stars.Services/Resources/AmazonCustomS3Config.cs @@ -26,6 +26,7 @@ internal void ForceCustomRegion(string region) internal void SetServiceURL(string serviceURL) { _serviceURL = serviceURL; + this.ServiceURL = _serviceURL; } } } \ No newline at end of file diff --git a/src/Stars.Services/Resources/S3ClientFactory.cs b/src/Stars.Services/Resources/S3ClientFactory.cs index b070c844..43468edb 100644 --- a/src/Stars.Services/Resources/S3ClientFactory.cs +++ b/src/Stars.Services/Resources/S3ClientFactory.cs @@ -350,11 +350,8 @@ private static AmazonCustomS3Config CreateS3Configuration(AWSOptions options) { config.RegionEndpoint = options.Region; } - else - { - config.SetServiceURL(defaultConfig.ServiceURL); - } } + config.SetServiceURL(defaultConfig.ServiceURL); return config; } @@ -380,7 +377,7 @@ public async Task GetWebIdentityCredentialsAsync(string serviceU var assumeRoleResult = await stsClient.AssumeRoleWithWebIdentityAsync(new AssumeRoleWithWebIdentityRequest { WebIdentityToken = jwt.RawData, - RoleArn = "arn:aws:iam::123456789012:role/RoleForTerradue", + // RoleArn = "arn:aws:iam::123456789012:role/RoleForTerradue", RoleSessionName = "MySession", DurationSeconds = 3600, Policy = policy diff --git a/src/Stars.Services/Resources/S3Options.cs b/src/Stars.Services/Resources/S3Options.cs index ff3b2ef0..ad153643 100644 --- a/src/Stars.Services/Resources/S3Options.cs +++ b/src/Stars.Services/Resources/S3Options.cs @@ -19,6 +19,8 @@ public S3Options() Policies = new S3OptionsPolicies(); } + public List DomainsWithNoCertificateVerification { get; set; } = new List(); + public Dictionary Services { get; set; } public S3OptionsPolicies Policies { get; set; } diff --git a/src/Stars.Services/Terradue.Stars.Services.csproj b/src/Stars.Services/Terradue.Stars.Services.csproj index 64855258..28a7b339 100644 --- a/src/Stars.Services/Terradue.Stars.Services.csproj +++ b/src/Stars.Services/Terradue.Stars.Services.csproj @@ -3,7 +3,7 @@ netstandard2.0 Terradue.Stars Stars is a set of services for working with Spatio Temporal Catalog such as STAC but not only - 2.19.0 + 2.19.2 $(Version)-$(VersionSuffix) Terradue.Stars.Services diff --git a/src/Stars.Tests/S3OptionsTests.cs b/src/Stars.Tests/S3OptionsTests.cs index fd11d7aa..26b2c69b 100644 --- a/src/Stars.Tests/S3OptionsTests.cs +++ b/src/Stars.Tests/S3OptionsTests.cs @@ -40,8 +40,6 @@ public async Task NamedS3ClientAsync() s3Client = s3ClientFactory.CreateS3Client(s3url); Assert.NotNull(s3Client); Assert.Equal("fr-par", s3Client.Config.AuthenticationRegion); - // the serviceUrl will be null as the new Custom config will use the DetermineServiceURL method - Assert.Equal(null, s3Client.Config.ServiceURL); } }