Skip to content

Commit

Permalink
Merge pull request #41 from Terradue/feature/saocom-again
Browse files Browse the repository at this point in the history 
Improved SAOCOM-1 metadata extraction
  • Loading branch information
@emmanuelmathot
emmanuelmathot authored Dec 11, 2023
2 parents 02be1c8 + 0347406 commit 57059a7
Show file tree
Hide file tree
Showing 8 changed files with 165 additions and 49 deletions.
16 changes: 9 additions & 7 deletions ...AOCOM-1/L1A/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
},
"properties": {
"sat:orbit_state": "ascending",
"saocom:path": 149,
"saocom:row": 447,
"proj:epsg": null,
"proj:shape": [
6419,
Expand Down Expand Up @@ -105,16 +107,16 @@
"roles": [
"metadata"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml"
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh.xml",
},
"amplitude-hh": {
"type": "image/tiff; application=geotiff",
"roles": [
"amplitude",
"data"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh",
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hh",
"sar:polarizations": [
"HH"
Expand All @@ -125,21 +127,21 @@
"roles": [
"metadata"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml"
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv.xml",
},
"amplitude-hv": {
"type": "image/tiff; application=geotiff",
"roles": [
"amplitude",
"data"
],
"href": "S1A_OPER_SAR_EOSSP__CORE_L1A_OLF_20211117T174240/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv",
"href": "data/286806-EOL1ASARSAO1A3111864/Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv",
"filename": "Data/slc-acqId0000588416-a-sm5-2111181029-s5dp-hv",
"sar:polarizations": [
"HV"
]
}
},
"links": []
}
}
22 changes: 12 additions & 10 deletions ...AOCOM-1/L1C/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1C_OLF_20200304T225242.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,17 +37,9 @@
]
},
"properties": {
"datetime": "2019-11-08T09:50:27.048Z",
"created": "2020-03-04T23:12:48Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"gsd": 10.0,
"title": "SAOCOM-1A SAO1A GEC VV/VH/HH/HV 2019-11-08 09:50:27",
"sat:orbit_state": "descending",
"saocom:path": 34,
"saocom:row": 151,
"proj:epsg": null,
"proj:shape": [
4463,
Expand All @@ -71,6 +63,16 @@
"sar:resolution_range": 10.0,
"sar:resolution_azimuth": 10.0,
"processing:level": "L1C",
"datetime": "2019-11-08T09:50:27.048Z",
"created": "2020-03-04T23:12:48Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"gsd": 10.0,
"title": "SAOCOM-1A SAO1A GEC VV/VH/HH/HV 2019-11-08 09:50:27",
"providers": [
{
"name": "CONAE",
Expand Down
20 changes: 11 additions & 9 deletions ...OCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20200819T231514.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,9 @@
]
},
"properties": {
"datetime": "2020-08-19T21:17:26.613Z",
"created": "2020-08-20T01:29:31Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH 2020-08-19 21:17:26",
"sat:orbit_state": "ascending",
"saocom:row": 529,
"saocom:path": 137,
"proj:epsg": null,
"proj:shape": [
9848,
Expand All @@ -68,6 +61,15 @@
"sar:resolution_range": 50.0,
"sar:resolution_azimuth": 50.0,
"processing:level": "L1D",
"datetime": "2020-08-19T21:17:26.613Z",
"created": "2020-08-20T01:29:31Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH 2020-08-19 21:17:26",
"providers": [
{
"name": "CONAE",
Expand Down
20 changes: 11 additions & 9 deletions ...OCOM-1/L1D/MetadataExtractorsTests_S1A_OPER_SAR_EOSSP__CORE_L1D_OLVF_20210204T093006.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,9 @@
]
},
"properties": {
"datetime": "2021-02-04T04:14:38.399Z",
"created": "2021-02-04T10:43:10Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH/HH/HV 2021-02-04 04:14:38",
"sat:orbit_state": "ascending",
"saocom:row": 406,
"saocom:path": 209,
"proj:epsg": null,
"proj:shape": [
4817,
Expand All @@ -70,6 +63,15 @@
"sar:resolution_range": 50.0,
"sar:resolution_azimuth": 50.0,
"processing:level": "L1D",
"datetime": "2021-02-04T04:14:38.399Z",
"created": "2021-02-04T10:43:10Z",
"platform": "saocom-1a",
"mission": "saocom-1",
"instruments": [
"sao1a"
],
"sensor_type": "radar",
"title": "SAOCOM-1A SAO1A GTC VV/VH/HH/HV 2021-02-04 04:14:38",
"providers": [
{
"name": "CONAE",
Expand Down
54 changes: 48 additions & 6 deletions src/Stars.Data/Model/Metadata/Saocom1/Saocom1MetadataExtractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ public class Saocom1MetadataExtractor : MetadataExtraction
{
public static XmlSerializer metadataSerializer = new XmlSerializer(typeof(SAOCOM_XMLProduct));
public static XmlSerializer manifestSerializer = new XmlSerializer(typeof(XEMT));
public static XmlSerializer parametersSerializer = new XmlSerializer(typeof(ParameterFile));
private readonly IFileSystem _fileSystem;
private readonly CarrierManager _carrierManager;

Expand Down Expand Up @@ -84,10 +85,14 @@ protected override async Task<StacNode> ExtractMetadata(IItem item, string suffi
if (zipAsset != null)
{
ZipArchiveAsset zipArchiveAsset = new ZipArchiveAsset(zipAsset, logger, resourceServiceProvider, _fileSystem);
zipArchiveAsset.IsInternalArchive = true;
zipArchiveAsset.UseParentAssetBaseDir = true;
var tmpDestination = LocalFileDestination.Create(_fileSystem.DirectoryInfo.FromDirectoryName(Path.GetTempPath()), item);

extractedAssets = await zipArchiveAsset.ExtractToDestinationAsync(tmpDestination, _carrierManager, System.Threading.CancellationToken.None);
string dirName = Path.GetDirectoryName(zipArchiveAsset.Uri.AbsolutePath) ?? Path.GetTempPath();
IResource route = item;
IDestination destination = LocalFileDestination.Create(_fileSystem.DirectoryInfo.FromDirectoryName(dirName), item, true);

extractedAssets = await zipArchiveAsset.ExtractToDestinationAsync(destination, _carrierManager, System.Threading.CancellationToken.None);
}

if (extractedAssets != null && item is IAssetsContainer)
Expand Down Expand Up @@ -129,6 +134,7 @@ internal virtual StacItem CreateStacItem(SAOCOM_XMLProduct metadata, XEMT manife
Dictionary<string, object> properties = new Dictionary<string, object>();
StacItem stacItem = new StacItem(ReadFilename(item), GetGeometry(item, kml, metadata), properties);
AddSatStacExtension(metadata, stacItem);
AddOrbitInformation(metadata, manifest, stacItem, item);
AddProjStacExtension(metadata, stacItem);
AddViewStacExtension(metadata, manifest, stacItem);
AddSarStacExtension(metadata, stacItem, item);
Expand Down Expand Up @@ -253,6 +259,37 @@ private void AddSatStacExtension(SAOCOM_XMLProduct metadata, StacItem stacItem)
sat.AbsoluteOrbit = absOrbit;
}

private void AddOrbitInformation(SAOCOM_XMLProduct metadata, XEMT manifest, StacItem stacItem, IItem item)
{
if (manifest != null && manifest.Product != null && manifest.Product.Features != null && manifest.Product.Features.GeographicAttributes != null && manifest.Product.Features.GeographicAttributes.PathRow != null)
{
stacItem.Properties["saocom:path"] = manifest.Product.Features.GeographicAttributes.PathRow.Path;
stacItem.Properties["saocom:row"] = manifest.Product.Features.GeographicAttributes.PathRow.Row;
}
else
{
IAsset parametersAsset = GetParametersAsset(item);
if (parametersAsset != null)
{
ParameterFile parameters = null;
using (var stream = resourceServiceProvider.GetAssetStreamAsync(parametersAsset, System.Threading.CancellationToken.None).Result)
{
var reader = XmlReader.Create(stream);
logger.LogDebug("Deserializing metadata file {0}", parametersAsset.Uri);
parameters = (ParameterFile)parametersSerializer.Deserialize(reader);
}
if (parameters.Inputs != null && parameters.Inputs.Parameters != null)
{
foreach (Parameter p in parameters.Inputs.Parameters)
{
if (p.Name == "Path" && Int32.TryParse(p.Value, out int path))stacItem.Properties["saocom:path"] = path;
if (p.Name == "Row" && Int32.TryParse(p.Value, out int row)) stacItem.Properties["saocom:row"] = row;
}
}
}
}
}

private string GetProductType(SAOCOM_XMLProduct metadata)
{
string fileName = metadata.Channel[0].RasterInfo.FileName;
Expand Down Expand Up @@ -634,21 +671,26 @@ private double GetGroundSampleDistance(SAOCOM_XMLProduct metadata, XEMT manifest

protected virtual IAsset GetMetadataAsset(IItem item)
{
IAsset metadataAsset = null;
metadataAsset = FindFirstAssetFromFileNameRegex(item, @"(slc|di|gec|gtc)-.*\.xml");
IAsset metadataAsset = FindFirstAssetFromFileNameRegex(item, @"(slc|di|gec|gtc)-.*\.xml");

return metadataAsset;
}


protected virtual IAsset GetManifestAsset(IItem item)
{
IAsset manifestAsset = null;
manifestAsset = FindFirstAssetFromFileNameRegex(item, @".*\.xemt");
IAsset manifestAsset = FindFirstAssetFromFileNameRegex(item, @".*\.xemt");
return manifestAsset;
}


protected virtual IAsset GetParametersAsset(IItem item)
{
IAsset parametersAsset = FindFirstAssetFromFileNameRegex(item, @"parameterFile2\.xml");
return parametersAsset;
}


protected virtual IAsset GetZipAsset(IItem item)
{
IAsset zipAsset = null;
Expand Down
60 changes: 59 additions & 1 deletion src/Stars.Data/Model/Metadata/Saocom1/Schema.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,7 +360,9 @@ public class SAOCOM_XMLProduct
public string At { get; set; }
}

// XEMT files (incomplete classes)
// =========================================================================
// .xemt files (incomplete classes)
// =========================================================================

[XmlRoot(ElementName = "xemt", Namespace = "http://www.conae.gov.ar/CUSS/XEMT")]
public class XEMT
Expand Down Expand Up @@ -410,6 +412,8 @@ public class Features
public Production Production { get; set; }
[XmlElement(ElementName = "acquisition")]
public Acquisition Acquisition { get; set; }
[XmlElement(ElementName = "geographicAttributes")]
public GeographicAttributes GeographicAttributes { get; set; }

}

Expand Down Expand Up @@ -456,5 +460,59 @@ public class AcquisitionParameters
public string SideLooking { get; set; }
}

[XmlRoot(ElementName = "geographicAttributes")]
public class GeographicAttributes
{
[XmlElement(ElementName = "pathRow")]
public PathRow PathRow { get; set; }
}

[XmlRoot(ElementName = "pathRow")]
public class PathRow
{
[XmlElement(ElementName = "Path")]
public int Path { get; set; }
[XmlElement(ElementName = "Row")]
public int Row { get; set; }
}

// =========================================================================
// parameters2.xml
// =========================================================================

[XmlRoot(ElementName = "parameters", Namespace = "http://www.conae.gov.ar/CGSS/XPNet")]
public class ParameterFile
{
[XmlElement(ElementName = "inputs")]
public Inputs Inputs { get; set; }
[XmlElement(ElementName = "outputs")]
public Outputs Outputs { get; set; }
}

[XmlRoot(ElementName = "inputs")]
public class Inputs
{
[XmlElement(ElementName = "parameter")]
public List<Parameter> Parameters { get; set; }
}

[XmlRoot(ElementName = "outputs")]
public class Outputs
{
[XmlElement(ElementName = "parameter")]
public List<Parameter> Parameters { get; set; }
}

[XmlRoot(ElementName = "parameter")]
public class Parameter
{
[XmlElement(ElementName = "name")]
public string Name { get; set; }
[XmlElement(ElementName = "type")]
public string Type { get; set; }
[XmlElement(ElementName = "value")]
public string Value { get; set; }
}


}
5 changes: 4 additions & 1 deletion src/Stars.Services/Processing/ZipArchiveAsset.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,9 @@ public bool UseParentAssetBaseDir
}
}

public bool IsInternalArchive { get; set; }


// Directory where ZIP file is located
// (value is set automatically if UseParentAssetBaseDir is set to true)
public string ParentAssetBaseDir { get; set; }
Expand All @@ -119,7 +122,7 @@ public async override Task<IAssetsContainer> ExtractToDestinationAsync(IDestinat
{
Dictionary<string, IAsset> assetsExtracted = new Dictionary<string, IAsset>();
zipFile = Ionic.Zip.ZipFile.Read(await GetZipStreamAsync(asset, carrierManager, ct));
string subFolder = AutodetectSubfolder();
string subFolder = IsInternalArchive ? String.Empty : AutodetectSubfolder();

foreach (var archiveAsset in Assets)
{
Expand Down
Loading

0 comments on commit 57059a7

Please sign in to comment.