diff --git a/packages/cogify/src/cogify/cli.ts b/packages/cogify/src/cogify/cli.ts index b3a6238d6..a038eaa2a 100644 --- a/packages/cogify/src/cogify/cli.ts +++ b/packages/cogify/src/cogify/cli.ts @@ -1,5 +1,6 @@ import { subcommands } from 'cmd-ts'; +import { TopoStacCreationCommand } from '../topo-raster/cli/cli.stac.js'; import { BasemapsCogifyCreateCommand } from './cli/cli.cog.js'; import { BasemapsCogifyConfigCommand } from './cli/cli.config.js'; import { BasemapsCogifyCoverCommand } from './cli/cli.cover.js'; @@ -12,5 +13,6 @@ export const CogifyCli = subcommands({ create: BasemapsCogifyCreateCommand, config: BasemapsCogifyConfigCommand, validate: BasemapsCogifyValidateCommand, + stac: TopoStacCreationCommand, }, }); diff --git a/packages/cogify/src/topo-raster/cli/cli.stac.ts b/packages/cogify/src/topo-raster/cli/cli.stac.ts new file mode 100644 index 000000000..52de85b6d --- /dev/null +++ b/packages/cogify/src/topo-raster/cli/cli.stac.ts @@ -0,0 +1,210 @@ +import { loadTiffsFromPaths } from '@basemaps/config-loader/build/json/tiff.config.js'; +import { Bounds, Epsg, Nztm2000Tms, TileMatrixSets } from '@basemaps/geo'; +import { fsa, LogType } from '@basemaps/shared'; +import { CliInfo } from '@basemaps/shared/build/cli/info.js'; +import { boolean, command, flag, option, string } from 'cmd-ts'; +import pLimit from 'p-limit'; + +import { isArgo } from '../../argo.js'; +import { UrlFolder } from '../../cogify/parsers.js'; +import { getLogger, logArguments } from '../../log.js'; +import { groupTiffsByDirectory } from '../mappers/group-tiffs-by-directory.js'; +import { mapEpsgToSlug } from '../mappers/map-epsg-to-slug.js'; +import { createStacCollection } from '../stac/create-stac-collection.js'; +import { createStacItems } from '../stac/create-stac-item-groups.js'; +import { writeStacFiles } from '../stac/write-stac-files.js'; +import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; + +const Q = pLimit(10); +export const brokenTiffs = { noBounds: [] as string[], noEpsg: [] as string[], noSize: [] as string[] }; + +/** + * List all the tiffs in a directory for topographic maps and create cogs for each. + * + * @param source: Location of the source files + * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ + * + * @param target: Location of the target path + */ +export const TopoStacCreationCommand = command({ + name: 'topo-stac-creation', + version: CliInfo.version, + description: 'List input topographic files, create StacItems, and generate tiles for grouping.', + args: { + ...logArguments, + title: option({ + type: string, + long: 'title', + description: 'Imported imagery title', + }), + source: option({ + type: UrlFolder, + long: 'source', + description: 'Location of the source files', + }), + target: option({ + type: UrlFolder, + long: 'target', + description: 'Target location for the output files', + }), + scale: option({ + type: string, + long: 'scale', + description: 'topo25, topo50, or topo250', + }), + resolution: option({ + type: string, + long: 'resolution', + description: 'e.g. gridless_600dpi', + }), + latestOnly: flag({ + type: boolean, + defaultValue: () => false, + long: 'latest-only', + description: 'Only process the latest version of each map sheet', + defaultValueIsSerializable: true, + }), + forceOutput: flag({ + type: boolean, + defaultValue: () => false, + long: 'force-output', + defaultValueIsSerializable: true, + }), + }, + async handler(args) { + const logger = getLogger(this, args); + const startTime = performance.now(); + logger.info('ListJobs:Start'); + + const { epsgDirectoryPaths, stacItemPaths } = await loadTiffsToCreateStacs( + args.latestOnly, + args.source, + args.target, + args.title, + args.scale, + args.resolution, + args.forceOutput, + logger, + ); + + if (epsgDirectoryPaths.length === 0 || stacItemPaths.length === 0) throw new Error('No Stac items created'); + + // write stac items into an JSON array + if (args.forceOutput || isArgo()) { + const targetURL = isArgo() ? new URL('/tmp/topo-stac-creation/') : args.target; + + // for create-config: we need to tell create-config to create a bundled config for each epsg folder (latest only). + // workflow: will loop 'targets.json' and create a node for each path where each node's job is to create a bundled config. + await fsa.write(new URL('targets.json', targetURL), JSON.stringify(epsgDirectoryPaths, null, 2)); + + // tiles.json makes the tiff files + await fsa.write(new URL('tiles.json', targetURL), JSON.stringify(stacItemPaths, null, 2)); + await fsa.write(new URL('brokenTiffs.json', targetURL), JSON.stringify(brokenTiffs, null, 2)); + } + + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); + }, +}); + +/** + * @param source: Source directory URL from which to load tiff files + * @example TODO + * + * @param target: Destination directory URL into which to save the STAC collection and item JSON files + * @example TODO + * + * @param title: The title of the collection + * @example "New Zealand Topo50 Map Series (Gridless)" + * + * @returns an array of StacItem objects + */ +async function loadTiffsToCreateStacs( + latestOnly: boolean, + source: URL, + target: URL, + title: string, + scale: string, + resolution: string, + forceOutput: boolean, + logger?: LogType, +): Promise<{ epsgDirectoryPaths: { epsg: string; url: URL }[]; stacItemPaths: { path: URL }[] }> { + logger?.info({ source }, 'LoadTiffs:Start'); + // extract all file paths from the source directory and convert them into URL objects + const fileURLs = await fsa.toArray(fsa.list(source)); + // process all of the URL objects into Tiff objects + const tiffs = await loadTiffsFromPaths(fileURLs, Q); + logger?.info({ numTiffs: tiffs.length }, 'LoadTiffs:End'); + + // group all of the Tiff objects by epsg and map code + logger?.info('GroupTiffs:Start'); + const itemsByDir = groupTiffsByDirectory(tiffs, logger); + const itemsByDirPath = new URL('itemsByDirectory.json', target); + await fsa.write(itemsByDirPath, JSON.stringify(itemsByDir, null, 2)); + logger?.info('GroupTiffs:End'); + + const epsgDirectoryPaths: { epsg: string; url: URL }[] = []; + const stacItemPaths = []; + + // create and write stac items and collections + for (const [epsg, itemsByMapCode] of itemsByDir.all.entries()) { + const allTargetURL = new URL(`${scale}/${resolution}/${epsg}/`, target); + const latestTargetURL = new URL(`${scale}_latest/${resolution}/${epsg}/`, target); + + const allBounds: Bounds[] = []; + const allStacItems: MapSheetStacItem[] = []; + + const latestBounds: Bounds[] = []; + const latestStacItems: MapSheetStacItem[] = []; + + // parse epsg + const epsgCode = Epsg.parse(epsg); + if (epsgCode == null) throw new Error(`Failed to parse epsg '${epsg}'`); + + // convert epsg to tile matrix + const tileMatrix = TileMatrixSets.tryGet(epsgCode) ?? Nztm2000Tms; // TODO: support other tile matrices + if (tileMatrix == null) throw new Error(`Failed to convert epsg code '${epsgCode.code}' to a tile matrix`); + + // create stac items + logger?.info({ epsg }, 'CreateStacItems:Start'); + for (const [mapCode, items] of itemsByMapCode.entries()) { + // get latest item + const latest = itemsByDir.latest.get(epsg).get(mapCode); + + // create stac items + const stacItems = createStacItems(allTargetURL, tileMatrix, items, latest, logger); + + allBounds.push(...items.map((item) => item.bounds)); + allStacItems.push(...stacItems.all); + + latestBounds.push(latest.bounds); + latestStacItems.push(stacItems.latest); + } + + // convert epsg to slug + const epsgSlug = mapEpsgToSlug(epsgCode.code); + if (epsgSlug == null) throw new Error(`Failed to map epsg code '${epsgCode.code}' to a slug`); + + const linzSlug = `${scale}-${epsgSlug}`; + + // create collections + const collection = createStacCollection(title, linzSlug, Bounds.union(allBounds), allStacItems, logger); + const latestCollection = createStacCollection(title, linzSlug, Bounds.union(latestBounds), latestStacItems, logger); + logger?.info({ epsg }, 'CreateStacItems:End'); + + if (forceOutput || isArgo()) { + epsgDirectoryPaths.push({ epsg, url: latestTargetURL }); + + // write stac items and collections + logger?.info({ epsg }, 'WriteStacFiles:Start'); + if (!latestOnly) { + const allPaths = await writeStacFiles(allTargetURL, allStacItems, collection, logger); + stacItemPaths.push(...allPaths.itemPaths); + } + const latestPaths = await writeStacFiles(latestTargetURL, latestStacItems, latestCollection, logger); + stacItemPaths.push(...latestPaths.itemPaths); + logger?.info({ epsg }, 'WriteStacFiles:End'); + } + } + + return { epsgDirectoryPaths, stacItemPaths }; +} diff --git a/packages/cogify/src/topo-raster/extractors/__test__/extract-map-code-and-version.test.ts b/packages/cogify/src/topo-raster/extractors/__test__/extract-map-code-and-version.test.ts new file mode 100644 index 000000000..a3066f9fd --- /dev/null +++ b/packages/cogify/src/topo-raster/extractors/__test__/extract-map-code-and-version.test.ts @@ -0,0 +1,37 @@ +import { strictEqual, throws } from 'node:assert'; +import { describe, it } from 'node:test'; + +import { extractMapCodeAndVersion } from '../extract-map-code-and-version.js'; + +describe('extractMapCodeAndVersion', () => { + const FakeDomain = 's3://topographic/fake-domain'; + const validFiles = [ + { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { + input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`, + expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' }, + }, + { + input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`, + expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' }, + }, + ]; + const invalidFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`]; + + it('should parse the correct MapSheet Names', () => { + for (const file of validFiles) { + const output = extractMapCodeAndVersion(file.input); + strictEqual(output.mapCode, file.expected.mapCode, 'Map code does not match'); + strictEqual(output.version, file.expected.version, 'Version does not match'); + } + }); + + it('should not able to parse a version from file', () => { + for (const file of invalidFiles) { + throws(() => extractMapCodeAndVersion(file), new Error('Version not found in the file name')); + } + }); +}); diff --git a/packages/cogify/src/topo-raster/extractors/extract-bounds-from-tiff.ts b/packages/cogify/src/topo-raster/extractors/extract-bounds-from-tiff.ts new file mode 100644 index 000000000..bf75871a6 --- /dev/null +++ b/packages/cogify/src/topo-raster/extractors/extract-bounds-from-tiff.ts @@ -0,0 +1,22 @@ +import { Bounds } from '@basemaps/geo'; +import { RasterTypeKey, Tiff, TiffTagGeo } from '@cogeotiff/core'; + +/** + * Attempts to extract a bounds set from the given Tiff object. + * + * @param tiff: the Tiff object from which to extract a bounds set. + * + * @returns a Bounds object, on success. Otherwise, null. + */ +export function extractBoundsFromTiff(tiff: Tiff): Bounds | null { + const img = tiff.images[0]; + if (img == null) { + throw new Error(`No images found in Tiff file: ${tiff.source.url.href}`); + } + + if (img.valueGeo(TiffTagGeo.GTRasterTypeGeoKey) === RasterTypeKey.PixelIsPoint) { + throw new Error("'Pixel is Point' raster grid spacing is not supported"); + } + + return Bounds.fromBbox(img.bbox); +} diff --git a/packages/cogify/src/topo-raster/extractors/extract-epsg-from-tiff.ts b/packages/cogify/src/topo-raster/extractors/extract-epsg-from-tiff.ts new file mode 100644 index 000000000..f814ae38b --- /dev/null +++ b/packages/cogify/src/topo-raster/extractors/extract-epsg-from-tiff.ts @@ -0,0 +1,50 @@ +import { Epsg } from '@basemaps/geo'; +import { LogType } from '@basemaps/shared'; +import { Tiff, TiffTagGeo } from '@cogeotiff/core'; + +const projections: Record = { + 'Universal Transverse Mercator Zone': Epsg.Wgs84, + 'Chatham Islands Transverse Mercator 2000': Epsg.Citm2000, + 'New Zealand Transverse Mercator 2000': Epsg.Nztm2000, +}; + +/** + * Attempts to extract an epsg value from the given Tiff object. + * + * @param tiff: The Tiff object from which to extract an epsg value. + * + * @returns an Epsg instance, on success. Otherwise, null. + */ +export function extractEpsgFromTiff(tiff: Tiff, logger?: LogType): Epsg | null { + const img = tiff.images[0]; + if (img == null) { + throw new Error(`No images found in Tiff file: ${tiff.source.url.href}`); + } + + // try to extract the epsg directly from the tiff + const epsg = img.epsg; + + if (epsg != null) { + const code = Epsg.tryGet(epsg); + + if (code != null) { + logger?.info({ found: true, method: 'direct' }, 'extractEpsgFromTiff()'); + return code; + } + } + + // try to extract the epsg from the tiff's projected citation geotag + const tag = img.valueGeo(TiffTagGeo.ProjectedCitationGeoKey); + + if (typeof tag === 'string') { + for (const [citation, epsg] of Object.entries(projections)) { + if (tag.startsWith(citation)) { + logger?.info({ found: true, method: 'geotag' }, 'extractEpsgFromTiff()'); + return epsg; + } + } + } + + logger?.info({ found: false }, 'extractEpsgFromTiff()'); + return null; +} diff --git a/packages/cogify/src/topo-raster/extractors/extract-map-code-and-version.ts b/packages/cogify/src/topo-raster/extractors/extract-map-code-and-version.ts new file mode 100644 index 000000000..f790f4ce7 --- /dev/null +++ b/packages/cogify/src/topo-raster/extractors/extract-map-code-and-version.ts @@ -0,0 +1,31 @@ +import { LogType } from '@basemaps/shared'; +import path from 'path'; + +/** + * Attempts to extract a map code and version from the filename of the provided filepath. + * Throws an error if either detail cannot be parsed. + * + * @param file: the filepath from which to extract a map code and version. + * + * @example + * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + * returns: { mapCode: "CJ10", version: "v1-00" } + * + * @returns an object containing the map code and version. + */ +export function extractMapCodeAndVersion(file: string, logger?: LogType): { mapCode: string; version: string } { + const url = new URL(file); + const filePath = path.parse(url.href); + const fileName = filePath.name; + + // extract map code from head of the file name (e.g. CJ10) + const mapCode = fileName.split('_')[0]; + if (mapCode == null) throw new Error('Map sheet not found in the file name'); + + // extract version from tail of the file name (e.g. v1-00) + const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; + if (version == null) throw new Error('Version not found in the file name'); + + logger?.info({ mapCode, version }, 'extractMapCodeAndVersion()'); + return { mapCode, version }; +} diff --git a/packages/cogify/src/topo-raster/extractors/extract-size-from-tiff.ts b/packages/cogify/src/topo-raster/extractors/extract-size-from-tiff.ts new file mode 100644 index 000000000..78d7c93e8 --- /dev/null +++ b/packages/cogify/src/topo-raster/extractors/extract-size-from-tiff.ts @@ -0,0 +1,22 @@ +import { Size } from '@basemaps/geo'; +import { LogType } from '@basemaps/shared'; +import { Tiff } from '@cogeotiff/core'; + +/** + * Attempts to extract a size from the given Tiff object. + * + * @param tiff: the Tiff object from which to extract the size. + * + * @returns a Size object, on success. Otherwise, null. + */ +export function extractSizeFromTiff(tiff: Tiff, logger?: LogType): Size | null { + try { + const size = tiff.images[0]?.size ?? null; + + logger?.info({ found: size }, 'extractSizeFromTiff()'); + return size; + } catch (e) { + logger?.info({ found: false }, 'extractSizeFromTiff()'); + return null; + } +} diff --git a/packages/cogify/src/topo-raster/mappers/group-tiffs-by-directory.ts b/packages/cogify/src/topo-raster/mappers/group-tiffs-by-directory.ts new file mode 100644 index 000000000..233741f7a --- /dev/null +++ b/packages/cogify/src/topo-raster/mappers/group-tiffs-by-directory.ts @@ -0,0 +1,81 @@ +import { LogType } from '@basemaps/shared'; +import { Tiff } from '@cogeotiff/core'; + +import { brokenTiffs } from '../cli/cli.stac.js'; +import { extractBoundsFromTiff as extractBoundsFromTiff } from '../extractors/extract-bounds-from-tiff.js'; +import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; +import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; +import { extractSizeFromTiff as extractSizeFromTiff } from '../extractors/extract-size-from-tiff.js'; +import { ByDirectory } from '../types/by-directory.js'; +import { TiffItem } from '../types/tiff-item.js'; + +/** + * Groups a list of Tiff objects into a directory-like structure + * based on each object's epsg, map code, and version information. + * + * This function assigns each tiff to a group based on its map code (e.g. "AT24"). + * For each group, it then identifies the latest version and sets a copy aside. * + * + * @param tiffs: the list of Tiff objects to group by epsg, and map code, and version + * @returns a `ByDirectory` promise + */ +export function groupTiffsByDirectory(tiffs: Tiff[], logger?: LogType): ByDirectory { + // group the tiffs by directory, epsg, and map code + const byDirectory = new ByDirectory(); + + // create items for each tiff and store them into 'all' by {epsg} and {map code} + for (const tiff of tiffs) { + const source = tiff.source.url; + const { mapCode, version } = extractMapCodeAndVersion(source.href, logger); + + const bounds = extractBoundsFromTiff(tiff); + const epsg = extractEpsgFromTiff(tiff, logger); + const size = extractSizeFromTiff(tiff, logger); + + if (bounds == null || epsg == null || size == null) { + if (bounds == null) { + brokenTiffs.noBounds.push(`${mapCode}_${version}`); + logger?.warn({ mapCode, version }, 'Could not extract bounds from tiff'); + } + + if (epsg == null) { + brokenTiffs.noEpsg.push(`${mapCode}_${version}`); + logger?.warn({ mapCode, version }, 'Could not extract epsg from tiff'); + } + + if (size == null) { + brokenTiffs.noSize.push(`${mapCode}_${version}`); + logger?.warn({ mapCode, version }, 'Could not extract width or height from tiff'); + } + + continue; + } + + const item = new TiffItem(tiff, source, mapCode, version, bounds, epsg, size); + + // push the item into 'all' by {epsg} and {map code} + byDirectory.all.get(epsg.toString()).get(mapCode, []).push(item); + } + + // for each {epsg} and {map code}, identify the latest item by {version} and copy it to 'latest' + for (const [epsg, byMapCode] of byDirectory.all.entries()) { + for (const [mapCode, items] of byMapCode.entries()) { + const sortedItems = items.sort((a, b) => a.version.localeCompare(b.version)); + + const latestItem = sortedItems[sortedItems.length - 1]; + if (latestItem == null) throw new Error(); + + // store the item into 'latest' by {epsg} and {map code} + byDirectory.latest.get(epsg).set(mapCode, latestItem); + } + } + + logger?.info( + byDirectory.all.entries().reduce((obj, [epsg, byMapCode]) => { + return { ...obj, [epsg]: byMapCode.entries().length }; + }, {}), + 'numItemsPerEpsg', + ); + + return byDirectory; +} diff --git a/packages/cogify/src/topo-raster/mappers/map-epsg-to-slug.ts b/packages/cogify/src/topo-raster/mappers/map-epsg-to-slug.ts new file mode 100644 index 000000000..5989ad187 --- /dev/null +++ b/packages/cogify/src/topo-raster/mappers/map-epsg-to-slug.ts @@ -0,0 +1,21 @@ +import { EpsgCode } from '@basemaps/geo'; +import { LogType } from '@basemaps/shared'; + +const slugs: { [key in EpsgCode]?: string } = { + [EpsgCode.Nztm2000]: 'new-zealand-mainland', + [EpsgCode.Citm2000]: 'chatham-islands', +}; + +/** + * Attempts to map the given EpsgCode enum to a slug. + * + * @param epsg: The EpsgCode enum to map to a slug + * + * @returns if succeeded, a slug string. Otherwise, null. + */ +export function mapEpsgToSlug(epsg: EpsgCode, logger?: LogType): string | null { + const slug = slugs[epsg]; + + logger?.info({ found: slug != null }, 'mapEpsgToSlug()'); + return slug ?? null; +} diff --git a/packages/cogify/src/topo-raster/stac/create-base-stac-item.ts b/packages/cogify/src/topo-raster/stac/create-base-stac-item.ts new file mode 100644 index 000000000..a27f04a75 --- /dev/null +++ b/packages/cogify/src/topo-raster/stac/create-base-stac-item.ts @@ -0,0 +1,83 @@ +import { TileMatrixSet } from '@basemaps/geo'; +import { LogType } from '@basemaps/shared'; +import { CliId, CliInfo } from '@basemaps/shared/build/cli/info.js'; +import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; + +import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; +import { TiffItem } from '../types/tiff-item.js'; + +const CLI_DATE = new Date().toISOString(); + +/** + * This function creates a base StacItem object based on the provided parameters. + * + * @param fileName: The map sheet's filename + * @example "CJ10" or "CJ10_v1-00" + * + * @param tiffItem TODO + * + * @returns a StacItem object + */ +export function createBaseStacItem( + fileName: string, + tiffItem: TiffItem, + tileMatrix: TileMatrixSet, + logger?: LogType, +): MapSheetStacItem { + logger?.info({ fileName }, 'createBaseStacItem()'); + + const item: MapSheetStacItem = { + type: 'Feature', + stac_version: '1.0.0', + id: fileName, + links: [ + { rel: 'self', href: `./${fileName}.json`, type: 'application/json' }, + { rel: 'collection', href: './collection.json', type: 'application/json' }, + { rel: 'parent', href: './collection.json', type: 'application/json' }, + { rel: 'linz_basemaps:source', href: tiffItem.source.href, type: 'image/tiff; application=geotiff' }, + ], + assets: { + source: { + href: tiffItem.source.href, + type: 'image/tiff; application=geotiff', + roles: ['data'], + }, + }, + stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], + properties: { + datetime: CLI_DATE, + map_code: tiffItem.mapCode, + version: tiffItem.version.replace('-', '.'), // e.g. "v1-00" to "v1.00" + 'proj:epsg': tiffItem.epsg.code, + 'source.width': tiffItem.size.width, + 'source.height': tiffItem.size.height, + 'linz_basemaps:options': { + // tileId: fileName, + tileMatrix: tileMatrix.identifier, + // preset: 'webp', + // blockSize: 512, + // bigTIFF: 'no', + // compression: 'webp', + // quality: 100, + // overviewCompress: 'webp', + // overviewQuality: 90, + // overviewResampling: 'lanczos', + sourceEpsg: tiffItem.epsg.code, + // addalpha: true, + // noReprojecting: true, + // srcwin: [0, 0, tiffItem.size.width - DEFAULT_TRIM_PIXEL_RIGHT, tiffItem.size.height], + }, + 'linz_basemaps:generated': { + package: CliInfo.package, + hash: CliInfo.hash, + version: CliInfo.version, + datetime: CLI_DATE, + }, + }, + geometry: { type: 'Polygon', coordinates: tiffItem.bounds.toPolygon() } as GeoJSONPolygon, + bbox: tiffItem.bounds.toBbox(), + collection: CliId, + }; + + return item; +} diff --git a/packages/cogify/src/topo-raster/stac/create-stac-collection.ts b/packages/cogify/src/topo-raster/stac/create-stac-collection.ts new file mode 100644 index 000000000..1714c1e39 --- /dev/null +++ b/packages/cogify/src/topo-raster/stac/create-stac-collection.ts @@ -0,0 +1,56 @@ +import { Bounds } from '@basemaps/geo'; +import { LogType } from '@basemaps/shared'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { StacCollection } from 'stac-ts'; + +import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; + +const cliDate = new Date().toISOString(); + +export function createStacCollection( + title: string, + linzSlug: string, + imageryBounds: Bounds, + items: MapSheetStacItem[], + logger?: LogType, +): StacCollection { + logger?.info({ items: items.length }, 'CreateStacCollection()'); + const collection: StacCollection = { + type: 'Collection', + stac_version: '1.0.0', + id: CliId, + title, + description: 'Topographic maps of New Zealand', + license: 'CC-BY-4.0', + links: [ + // TODO: We not have an ODR bucket for the linz-topographic yet. + // { + // rel: 'root', + // href: 'https://nz-imagery.s3.ap-southeast-2.amazonaws.com/catalog.json', + // type: 'application/json', + // }, + { rel: 'self', href: './collection.json', type: 'application/json' }, + ...items.map((item) => { + return { + href: `./${item.id}.json`, + rel: 'item', + type: 'application/json', + }; + }), + ], + providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], + 'linz:lifecycle': 'ongoing', + 'linz:geospatial_category': 'topographic-maps', + 'linz:region': 'new-zealand', + 'linz:security_classification': 'unclassified', + 'linz:slug': linzSlug, + extent: { + spatial: { bbox: [imageryBounds.toBbox()] }, + // Default the temporal time today if no times were found as it is required for STAC + temporal: { interval: [[cliDate, null]] }, + }, + stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], + }; + + return collection; +} diff --git a/packages/cogify/src/topo-raster/stac/create-stac-item-groups.ts b/packages/cogify/src/topo-raster/stac/create-stac-item-groups.ts new file mode 100644 index 000000000..6514931e8 --- /dev/null +++ b/packages/cogify/src/topo-raster/stac/create-stac-item-groups.ts @@ -0,0 +1,48 @@ +import { TileMatrixSet } from '@basemaps/geo'; +import { LogType } from '@basemaps/shared'; + +import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; +import { TiffItem } from '../types/tiff-item.js'; +import { createBaseStacItem } from './create-base-stac-item.js'; + +/** + * This function needs to create two groups: + * - StacItem objects that will live in the "topo[50|250]" directory + * - StacItem objects that will live in the "topo[50|250]_latest" directory + * + * All versions need a StacItem object that will live in the topo[50/250] directory + * The latest version needs a second StacItem object that will live in the topo[50|250]_latest directory + */ +export function createStacItems( + allTargetURL: URL, + tileMatrix: TileMatrixSet, + all: TiffItem[], + latest: TiffItem, + logger?: LogType, +): { all: MapSheetStacItem[]; latest: MapSheetStacItem } { + const allStacItems = all.map((item) => + createBaseStacItem(`${item.mapCode}_${item.version}`, item, tileMatrix, logger), + ); + + const latestURL = new URL(`${latest.mapCode}_${latest.version}.json`, allTargetURL); + + // add link to all items pointing to the latest version + allStacItems.forEach((stacItem) => { + stacItem.links.push({ + href: latestURL.href, + rel: 'latest-version', + type: 'application/json', + }); + }); + + const latestStacItem = createBaseStacItem(latest.mapCode, latest, tileMatrix); + + // add link to the latest item referencing its copy that will live in the topo[50/250] directory + latestStacItem.links.push({ + href: latestURL.href, + rel: 'derived_from', + type: 'application/json', + }); + + return { latest: latestStacItem, all: allStacItems }; +} diff --git a/packages/cogify/src/topo-raster/stac/write-stac-files.ts b/packages/cogify/src/topo-raster/stac/write-stac-files.ts new file mode 100644 index 000000000..6f552c4bb --- /dev/null +++ b/packages/cogify/src/topo-raster/stac/write-stac-files.ts @@ -0,0 +1,29 @@ +import { fsa, LogType } from '@basemaps/shared'; +import { StacCollection } from 'stac-ts'; + +import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; + +export async function writeStacFiles( + target: URL, + items: MapSheetStacItem[], + collection: StacCollection, + logger?: LogType, +): Promise<{ itemPaths: { path: URL }[]; collectionPath: URL }> { + // Create collection json for all topo50-latest items. + logger?.info({ target }, 'CreateStac:Output'); + logger?.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); + + const itemPaths = []; + + for (const item of items) { + const itemPath = new URL(`${item.id}.json`, target); + itemPaths.push({ path: itemPath }); + + await fsa.write(itemPath, JSON.stringify(item, null, 2)); + } + + const collectionPath = new URL('collection.json', target); + await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); + + return { itemPaths, collectionPath }; +} diff --git a/packages/cogify/src/topo-raster/types/by-directory.ts b/packages/cogify/src/topo-raster/types/by-directory.ts new file mode 100644 index 000000000..9b51e7fa2 --- /dev/null +++ b/packages/cogify/src/topo-raster/types/by-directory.ts @@ -0,0 +1,110 @@ +// ByDirectory { +// all: { +// [epsg: string]: { +// [mapCode: string]: T[] +// } +// }, +// latest: { +// [epsg: string]: { +// [mapCode: string]: T +// } +// } +// } +export class ByDirectory { + readonly all: ByEpsg; + readonly latest: ByEpsg; + + constructor() { + this.all = new ByEpsg(); + this.latest = new ByEpsg(); + } + + toJSON(): object { + return { + all: this.all.toJSON(), + latest: this.latest.toJSON(), + }; + } +} + +// ByEpsg { +// [epsg: string]: { +// [mapCode: string]: T +// } +// } +class ByEpsg { + private readonly items: { [epsg: string]: ByMapCode }; + + constructor() { + this.items = {}; + } + + get(epsg: string): ByMapCode { + let result = this.items[epsg]; + + if (result !== undefined) return result; + + result = new ByMapCode(); + this.items[epsg] = result; + + return result; + } + + /** + * @returns [epsg, ByMapCode][] + */ + entries(): [string, ByMapCode][] { + return Object.entries(this.items); + } + + toJSON(): object { + return Object.entries(this.items).reduce((obj, [epsg, byMapCode]) => ({ ...obj, [epsg]: byMapCode.toJSON() }), {}); + } +} + +// ByMapCode { +// [mapCode: string]: T +// } +class ByMapCode { + private readonly items: { [mapCode: string]: T }; + + constructor() { + this.items = {}; + } + + /** + * @param mapCode: the map code to lookup + * @param defaultValue: the value to set and return if no value is found for the given map code + * + * @returns the value for the given map code, otherwise, the default value + * @throws {Error} if no value is found for the given map code and no default value is provided. + */ + get(mapCode: string, defaultValue?: T): T { + let result = this.items[mapCode]; + + if (result !== undefined) return result; + if (defaultValue === undefined) { + throw new Error(`No value found for map code '${mapCode}'`); + } + + result = defaultValue; + this.items[mapCode] = result; + + return result; + } + + set(mapCode: string, value: T): void { + this.items[mapCode] = value; + } + + /** + * @returns [mapCode, T][] + */ + entries(): [string, T][] { + return Object.entries(this.items); + } + + toJSON(): object { + return Object.entries(this.items).reduce((obj, [mapCode, value]) => ({ ...obj, [mapCode]: value }), {}); + } +} diff --git a/packages/cogify/src/topo-raster/types/map-sheet-stac-item.ts b/packages/cogify/src/topo-raster/types/map-sheet-stac-item.ts new file mode 100644 index 000000000..17197799b --- /dev/null +++ b/packages/cogify/src/topo-raster/types/map-sheet-stac-item.ts @@ -0,0 +1,83 @@ +import { EpsgCode } from '@basemaps/geo'; +import { StacAsset, StacItem } from 'stac-ts'; + +export interface MapSheetStacItem extends StacItem { + assets: { + /** + * A StacAsset describing a map sheet's source file. + * + * @example { + * href: 's3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-03.tif', + * type: 'image/tiff; application=geotiff', + * roles: ['data'] + * 'file:size': 144699034 (added during COG creation) + * 'file:checksum': '12207e3a289637b0bbb921ad0c3a6404ce6b0149330cdc675d520ce9d690b7792d52' (added during COG creation) + * } + */ + source: StacAsset; + + /** + * A StacAsset describing a map sheet's cloud-optimised file (added after COG creation). + * + * @example { + * href: './CJ10.tiff', + * type: 'image/tiff; application=geotiff; profile=cloud-optimized', + * roles: ['data'], + * 'file:size': 2707038 + * 'file:checksum': '12204ea0ee43553e849b46a0f8b3281a532d855bac19e9d8845ea229056b568eccb5' + * } + */ + cog?: StacAsset; + }; + properties: { + /** + * An ISO string representing a Stac Item's creation time. + * + * @pattern `YYYY-MM-DDTHH:mm:ss.sssZ` + * + * @example "2024-01-31T09:41:12.345Z" + */ + datetime: string; + + /** + * A map sheet's code. + * + * @example "CJ10" + */ + map_code: string; + + /** + * A map sheet's version. + * + * @example "v1.00" + */ + version: string; + + /** + * An EpsgCode Enum representing a map sheet's projection. + * + * @example EpsgCode.Nztm2000 = 2193 + */ + 'proj:epsg': EpsgCode; + + /** + * The width of a map sheet in pixels. + */ + 'source.width': number; + + /** + * The height of a map sheet in pixels. + */ + 'source.height': number; + + /** + * An object of key-value pairs describing options for Basemaps' cogify process. + */ + 'linz_basemaps:options': { [key: string]: unknown }; + + /** + * An object of key-value pairs information for basemaps cli packages. + */ + 'linz_basemaps:generated': { [key: string]: unknown }; + }; +} diff --git a/packages/cogify/src/topo-raster/types/tiff-item.ts b/packages/cogify/src/topo-raster/types/tiff-item.ts new file mode 100644 index 000000000..a3ca8907f --- /dev/null +++ b/packages/cogify/src/topo-raster/types/tiff-item.ts @@ -0,0 +1,26 @@ +import { Bounds, Epsg, Size } from '@basemaps/geo'; +import { Tiff } from '@basemaps/shared'; + +export class TiffItem { + readonly tiff: Tiff; + readonly source: URL; + readonly mapCode: string; + readonly version: string; + readonly bounds: Bounds; + readonly epsg: Epsg; + readonly size: Size; + + constructor(tiff: Tiff, source: URL, mapCode: string, version: string, bounds: Bounds, epsg: Epsg, size: Size) { + this.tiff = tiff; + this.source = source; + this.mapCode = mapCode; + this.version = version; + this.bounds = bounds; + this.epsg = epsg; + this.size = size; + } + + toJSON(): string { + return `${this.mapCode}_${this.version}`; + } +}