feat(basemaps): New cli listing jobs for topo raster map standardised workflow. BM-1127

package.json

@@ -39,7 +39,9 @@
     "@aws-sdk/credential-providers": "^3.438.0",
     "@aws-sdk/lib-storage": "^3.440.0",
     "@basemaps/config": "^7.7.0",
+    "@basemaps/config-loader": "^7.12.0",
     "@basemaps/geo": "^7.5.0",
+    "@basemaps/shared": "^7.12.0",
     "@chunkd/fs": "^10.0.9",
     "@chunkd/source-aws-v3": "^10.1.3",
     "@cogeotiff/core": "^9.0.3",

src/cli.info.ts

@@ -1,3 +1,5 @@
+import * as ulid from 'ulid';
+
 export const CliInfo = {
   package: '@linzjs/argo-tasks',
   // Git version information
@@ -7,3 +9,6 @@ export const CliInfo = {
   // Github action that the CLI was built from
   buildId: process.env['GITHUB_RUN_ID'] ? `${process.env['GITHUB_RUN_ID']}-${process.env['GITHUB_RUN_ATTEMPT']}` : '',
 };
+
+/** Unique Id for this instance of the cli being run */
+export const CliId = ulid.ulid();

src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts

@@ -0,0 +1,37 @@
+import assert from 'node:assert';
+import { describe, it } from 'node:test';
+
+import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js';
+
+describe('extractMapCodeAndVersion', () => {
+  const FakeDomain = 's3://topographic/fake-domain';
+  const FakeFiles = [
+    { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } },
+    {
+      input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' },
+    },
+    {
+      input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`,
+      expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' },
+    },
+  ];
+
+  it('Should parse the correct MapSheet Names', async () => {
+    for (const file of FakeFiles) {
+      const output = extractMapCodeAndVersion(file.input);
+      assert.equal(output.mapCode, file.expected.mapCode, 'Map code does not match');
+      assert.equal(output.version, file.expected.version, 'Version does not match');
+    }
+  });
+
+  it('Should not able to parse a version from file', async () => {
+    const wrongFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`];
+    for (const file of wrongFiles) {
+      assert.throws(() => extractMapCodeAndVersion(file), new Error('Version not found in the file name'));
+    }
+  });
+});

src/commands/basemaps-topo-import/extractors/extract-bounds-from-tiff.ts

@@ -0,0 +1,24 @@
+import { Bounds } from '@basemaps/geo';
+import { Tiff } from '@cogeotiff/core';
+
+import { logger } from '../../../log.js';
+import { findBoundingBox } from '../../../utils/geotiff.js';
+
+/**
+ * Attempts to extract bounds from the given Tiff object.
+ *
+ * @param tiff: The Tiff object from which to extract bounds
+ *
+ * @returns if succeeded, a Bounds object. Otherwise, null.
+ */
+export async function extractBoundsFromTiff(tiff: Tiff): Promise<Bounds | null> {
+  try {
+    const bounds = Bounds.fromBbox(await findBoundingBox(tiff));
+
+    logger.info({ found: true }, 'extractBoundsFromTiff()');
+    return bounds;
+  } catch (e) {
+    logger.info({ found: false }, 'extractBoundsFromTiff()');
+    return null;
+  }
+}

src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts

@@ -0,0 +1,35 @@
+import { Epsg } from '@basemaps/geo';
+import { Tiff, TiffTagGeo } from '@cogeotiff/core';
+
+import { logger } from '../../../log.js';
+import { extractEpsg } from '../../generate-path/path.generate.js';
+
+const projections = [
+  ['Universal Transverse Mercator Zone', Epsg.Wgs84],
+  ['Chatham Islands Transverse Mercator 2000', Epsg.Citm2000],
+  ['New Zealand Transverse Mercator 2000', Epsg.Nztm2000],
+] as const;
+
+export function extractEpsgFromTiff(tiff: Tiff): Epsg | null {
+  // try to extract the epsg directly from the tiff
+  try {
+    const epsg = Epsg.get(extractEpsg(tiff));
+    if (epsg != null) {
+      logger.info({ found: epsg.code }, 'extractEpsgFromTiff()');
+      return epsg;
+    }
+  } catch {
+    // try to extract the epsg from the tiff's projected citation geotag
+    const tag = tiff.images[0]?.valueGeo(TiffTagGeo.ProjectedCitationGeoKey);
+
+    for (const [citation, epsg] of projections) {
+      if (tag?.startsWith(citation)) {
+        logger.info({ found: epsg.code }, 'extractEpsgFromTiff()');
+        return epsg;
+      }
+    }
+  }
+
+  logger.info({ found: false }, 'extractEpsgFromTiff()');
+  return null;
+}

src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts

@@ -0,0 +1,33 @@
+import path from 'path';
+
+import { logger } from '../../../log.js';
+import { tryParseUrl } from '../../common.js';
+
+/**
+ * Extract the map code and version from the provided path.
+ * Throws an error if either detail cannot be parsed.
+ *
+ * @param file: filepath from which to extract the map code and version
+ *
+ * @example
+ * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif"
+ * returns: { mapCode: "CJ10", version: "v1-00" }
+ *
+ * @returns an object containing the map code and version
+ */
+export function extractMapCodeAndVersion(file: string): { mapCode: string; version: string } {
+  const url = tryParseUrl(file);
+  const filePath = path.parse(url.href);
+  const fileName = filePath.name;
+
+  // extract map code from head of the file name (e.g. CJ10)
+  const mapCode = fileName.split('_')[0];
+  if (mapCode == null) throw new Error('Map sheet not found in the file name');
+
+  // extract version from tail of the file name (e.g. v1-00)
+  const version = fileName.match(/v(\d)-(\d\d)/)?.[0];
+  if (version == null) throw new Error('Version not found in the file name');
+
+  logger.info({ mapCode, version }, 'extractMapCodeAndVersion()');
+  return { mapCode, version };
+}

src/commands/basemaps-topo-import/extractors/extract-size-from-tiff.ts

@@ -0,0 +1,23 @@
+import { Size } from '@basemaps/geo';
+import { Tiff } from '@cogeotiff/core';
+
+import { logger } from '../../../log.js';
+
+/**
+ *  Attempts to extract a size from the given Tiff object.
+ *
+ * @param tiff: The Tiff object from which to extract the size
+ *
+ * @returns if succeeded, a Size object. Otherwise, null.
+ */
+export function extractSizeFromTiff(tiff: Tiff): Size | null {
+  try {
+    const size = tiff.images[0]?.size ?? null;
+
+    logger.info({ found: true }, 'extractSizeFromTiff()');
+    return size;
+  } catch (e) {
+    logger.info({ found: false }, 'extractSizeFromTiff()');
+    return null;
+  }
+}

src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts

@@ -0,0 +1,79 @@
+import { Tiff } from '@cogeotiff/core';
+
+import { logger } from '../../../log.js';
+import { extractBoundsFromTiff as extractBoundsFromTiff } from '../extractors/extract-bounds-from-tiff.js';
+import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js';
+import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js';
+import { extractSizeFromTiff as extractSizeFromTiff } from '../extractors/extract-size-from-tiff.js';
+import { brokenTiffs } from '../topo-stac-creation.js';
+import { ByDirectory } from '../types/by-directory.js';
+import { TiffItem } from '../types/tiff-item.js';
+
+/**
+ * We need to assign each tiff to a group based on its map code (e.g. "AT24").
+ * For each group, we then need to identify the latest version and set it aside from the rest.
+ * The latest version will have special metadata, whereas the rest will have similar metadata.
+ *
+ * @param tiffs: The tiffs to group by epsg, and map code
+ * @returns a `ByDirectory<TiffItem>` promise
+ */
+export async function groupTiffsByDirectory(tiffs: Tiff[]): Promise<ByDirectory<TiffItem>> {
+  // group the tiffs by directory, epsg, and map code
+  const byDirectory = new ByDirectory<TiffItem>();
+
+  // create items for each tiff and store them into 'all' by {epsg} and {map code}
+  for (const tiff of tiffs) {
+    const source = tiff.source.url;
+    const { mapCode, version } = extractMapCodeAndVersion(source.href);
+
+    const bounds = await extractBoundsFromTiff(tiff);
+    const epsg = extractEpsgFromTiff(tiff);
+    const size = extractSizeFromTiff(tiff);
+
+    if (bounds == null || epsg == null || size == null) {
+      if (bounds == null) {
+        brokenTiffs.noBounds.push(`${mapCode}_${version}`);
+        logger.warn({ mapCode, version }, 'Could not extract bounds from tiff');
+      }
+
+      if (epsg == null) {
+        brokenTiffs.noEpsg.push(`${mapCode}_${version}`);
+        logger.warn({ mapCode, version }, 'Could not extract epsg from tiff');
+      }
+
+      if (size == null) {
+        brokenTiffs.noSize.push(`${mapCode}_${version}`);
+        logger.warn({ mapCode, version }, 'Could not extract width or height from tiff');
+      }
+
+      continue;
+    }
+
+    const item = new TiffItem(tiff, source, mapCode, version, bounds, epsg, size);
+
+    // push the item into 'all' by {epsg} and {map code}
+    byDirectory.all.get(epsg.toString()).get(mapCode, []).push(item);
+  }
+
+  // for each {epsg} and {map code}, identify the latest item by {version} and copy it to 'latest'
+  for (const [epsg, byMapCode] of byDirectory.all.entries()) {
+    for (const [mapCode, items] of byMapCode.entries()) {
+      const sortedItems = items.sort((a, b) => a.version.localeCompare(b.version));
+
+      const latestItem = sortedItems[sortedItems.length - 1];
+      if (latestItem == null) throw new Error();
+
+      // store the item into 'latest' by {epsg} and {map code}
+      byDirectory.latest.get(epsg).set(mapCode, latestItem);
+    }
+  }
+
+  logger.info(
+    byDirectory.all.entries().reduce((obj, [epsg, byMapCode]) => {
+      return { ...obj, [epsg]: byMapCode.entries().length };
+    }, {}),
+    'numItemsPerEpsg',
+  );
+
+  return byDirectory;
+}

src/commands/basemaps-topo-import/mappers/map-epsg-to-slug.ts

@@ -0,0 +1,27 @@
+import { EpsgCode } from '@basemaps/geo';
+
+import { logger } from '../../../log.js';
+
+const slugs: { [key in EpsgCode]?: string } = {
+  [EpsgCode.Nztm2000]: 'new-zealand-mainland',
+  [EpsgCode.Citm2000]: 'chatham-islands',
+};
+
+/**
+ * Attempts to map the given EpsgCode enum to a slug.
+ *
+ * @param epsg: The EpsgCode enum to map to a slug
+ *
+ * @returns if succeeded, a slug string. Otherwise, null.
+ */
+export function mapEpsgToSlug(epsg: EpsgCode): string | null {
+  const slug = slugs[epsg];
+
+  if (slug == null) {
+    logger.info({ found: false }, 'mapEpsgToSlug()');
+    return null;
+  }
+
+  logger.info({ found: true }, 'mapEpsgToSlug()');
+  return slug;
+}

src/commands/basemaps-topo-import/stac/create-base-stac-item.ts

@@ -0,0 +1,79 @@
+import { TileMatrixSet } from '@basemaps/geo';
+import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js';
+
+import { CliId, CliInfo } from '../../../cli.info.js';
+import { logger } from '../../../log.js';
+import { MapSheetStacItem } from '../types/map-sheet-stac-item.js';
+import { TiffItem } from '../types/tiff-item.js';
+
+const CLI_DATE = new Date().toISOString();
+const DEFAULT_TRIM_PIXEL_RIGHT = 1.7;
+
+/**
+ * This function creates a base StacItem object based on the provided parameters.
+ *
+ * @param fileName: The map sheet's filename
+ * @example "CJ10" or "CJ10_v1-00"
+ *
+ * @param tiffItem TODO
+ *
+ * @returns a StacItem object
+ */
+export function createBaseStacItem(fileName: string, tiffItem: TiffItem, tileMatrix: TileMatrixSet): MapSheetStacItem {
+  logger.info({ fileName }, 'createBaseStacItem()');
+
+  const item: MapSheetStacItem = {
+    type: 'Feature',
+    stac_version: '1.0.0',
+    id: fileName,
+    links: [
+      { rel: 'self', href: `./${fileName}.json`, type: 'application/json' },
+      { rel: 'collection', href: './collection.json', type: 'application/json' },
+      { rel: 'parent', href: './collection.json', type: 'application/json' },
+      { rel: 'linz_basemaps:source', href: tiffItem.source.href, type: 'image/tiff; application=geotiff' },
+    ],
+    assets: {
+      source: {
+        href: tiffItem.source.href,
+        type: 'image/tiff; application=geotiff',
+        roles: ['data'],
+      },
+    },
+    stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'],
+    properties: {
+      datetime: CLI_DATE,
+      map_code: tiffItem.mapCode,
+      version: tiffItem.version.replace('-', '.'), // e.g. "v1-00" to "v1.00"
+      'proj:epsg': tiffItem.epsg.code,
+      'source.width': tiffItem.size.width,
+      'source.height': tiffItem.size.height,
+      'linz_basemaps:options': {
+        tileId: fileName,
+        tileMatrix: tileMatrix.identifier,
+        preset: 'webp',
+        blockSize: 512,
+        bigTIFF: 'no',
+        compression: 'webp',
+        quality: 100,
+        overviewCompress: 'webp',
+        overviewQuality: 90,
+        overviewResampling: 'lanczos',
+        sourceEpsg: tiffItem.epsg.code,
+        addalpha: true,
+        noReprojecting: true,
+        srcwin: [0, 0, tiffItem.size.width - DEFAULT_TRIM_PIXEL_RIGHT, tiffItem.size.height],
+      },
+      'linz_basemaps:generated': {
+        package: CliInfo.package,
+        hash: CliInfo.hash,
+        version: CliInfo.version,
+        datetime: CLI_DATE,
+      },
+    },
+    geometry: { type: 'Polygon', coordinates: tiffItem.bounds.toPolygon() } as GeoJSONPolygon,
+    bbox: tiffItem.bounds.toBbox(),
+    collection: CliId,
+  };
+
+  return item;
+}