From 934394c0ebb3805caba63c69343ad46e8dd8e29c Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Fri, 22 Nov 2024 17:11:35 +1300 Subject: [PATCH 01/47] WIP --- .../basemaps-topo-import/list-jobs.ts | 70 +++++++++++++++++++ src/commands/index.ts | 2 + 2 files changed, 72 insertions(+) create mode 100644 src/commands/basemaps-topo-import/list-jobs.ts diff --git a/src/commands/basemaps-topo-import/list-jobs.ts b/src/commands/basemaps-topo-import/list-jobs.ts new file mode 100644 index 00000000..4c434de6 --- /dev/null +++ b/src/commands/basemaps-topo-import/list-jobs.ts @@ -0,0 +1,70 @@ +import { fsa } from '@chunkd/fs'; +import { command, option, string } from 'cmd-ts'; + +import { CliInfo } from '../../cli.info.js'; +import { logger } from '../../log.js'; +import { isArgo } from '../../utils/argo.js'; +import { config, forceOutput, registerCli, verbose } from '../common.js'; + +type Output = { + output: string; + input: string[]; +}; + +/** + * List all the tiffs in a directory and output the list of names to process. + * Outputs a json file with the list of files to process. + * + * @example + * [{ + * "output": "CJ10_v1-00", + * "input": [ + * "s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + * ] + * }] + * + * @param location: Location of the source files + * @example s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ + */ +export const basemapsListTopoJobs = command({ + name: 'list-topo-jobs', + description: 'List input files and validate there are no duplicates.', + version: CliInfo.version, + args: { + config, + verbose, + forceOutput, + location: option({ + type: string, + long: 'location', + description: 'Location of the source files', + }), + }, + async handler(args) { + const startTime = performance.now(); + registerCli(this, args); + logger.info('ListJobs:Start'); + + const files = await fsa.toArray(fsa.list(args.location)); + const outputs: Output[] = []; + + for (const file of files) { + logger.info({ file }, 'ListJobs:File'); + + if (file.endsWith('.tif') || file.endsWith('.tiff')) { + const fileUrl = new URL(file); + const tileName = fileUrl.pathname.split('/').pop(); + if (tileName == null) throw new Error(`Cannot get the tile name from file ${file}`); + + //Convert to the output filename + const output = tileName.replace('.tif', '').replace('GRIDLESS_GeoTif', ''); + logger.info({ output }, 'ListJobs:Output'); + outputs.push({ output, input: [file] }); + } + } + + if (args.forceOutput || isArgo()) + await fsa.write('/tmp/list-topo-jobs/file-list.json', JSON.stringify(outputs, null, 2)); + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); + }, +}); diff --git a/src/commands/index.ts b/src/commands/index.ts index 42b70b35..9773af1d 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -3,6 +3,7 @@ import { subcommands } from 'cmd-ts'; import { CliInfo } from '../cli.info.js'; import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; +import { basemapsListTopoJobs } from './basemaps-topo-import/list-jobs.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; import { commandGeneratePath } from './generate-path/path.generate.js'; @@ -48,6 +49,7 @@ export const AllCommands = { cmds: { 'create-pr': basemapsCreatePullRequest, 'create-mapsheet': basemapsCreateMapSheet, + 'list-topo-jobs': basemapsListTopoJobs, }, }), 'pretty-print': commandPrettyPrint, From fb1ab614963bafd206194814f2c3a7ccc9a31f71 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Mon, 25 Nov 2024 09:36:11 +1300 Subject: [PATCH 02/47] New topo-list-job commands to list the jobs for topo raster import --- src/commands/basemaps-topo-import/list-jobs.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/commands/basemaps-topo-import/list-jobs.ts b/src/commands/basemaps-topo-import/list-jobs.ts index 4c434de6..2bdedb05 100644 --- a/src/commands/basemaps-topo-import/list-jobs.ts +++ b/src/commands/basemaps-topo-import/list-jobs.ts @@ -63,6 +63,8 @@ export const basemapsListTopoJobs = command({ } } + if (outputs.length === 0) throw new Error('No tiff files found in the location'); + if (args.forceOutput || isArgo()) await fsa.write('/tmp/list-topo-jobs/file-list.json', JSON.stringify(outputs, null, 2)); logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); From 722fe7422bcb4af994189e8dcd8af7de0ceb85fd Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Mon, 25 Nov 2024 11:19:37 +1300 Subject: [PATCH 03/47] Minor fixes and unit tests. --- .../__test__/list-jobs.test.ts | 30 ++++++++++ .../basemaps-topo-import/list-jobs.ts | 59 +++++++++++++------ 2 files changed, 70 insertions(+), 19 deletions(-) create mode 100644 src/commands/basemaps-topo-import/__test__/list-jobs.test.ts diff --git a/src/commands/basemaps-topo-import/__test__/list-jobs.test.ts b/src/commands/basemaps-topo-import/__test__/list-jobs.test.ts new file mode 100644 index 00000000..753a9950 --- /dev/null +++ b/src/commands/basemaps-topo-import/__test__/list-jobs.test.ts @@ -0,0 +1,30 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +import { extractMapSheetName } from '../list-jobs.js'; + +describe('extractMapSheetName', () => { + const FakeDomain = 's3://topographic/fake-domain'; + const FakeFiles = [ + { actual: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: 'MB07_v1-00' }, + { actual: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: 'MB07_v1-00' }, + { actual: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: 'MB07_v1-00' }, + { actual: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: 'MB07_v1-00' }, + { actual: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`, expected: 'AX32ptsAX31AY31AY32_v1-00' }, + { actual: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`, expected: 'AZ36ptsAZ35BA35BA36_v1-00' }, + ]; + + it('Should parse the correct MapSheet Names', async () => { + for (const file of FakeFiles) { + const output = extractMapSheetName(file.actual); + assert.equal(output, file.expected); + } + }); + + it('Should not able to parse a version from file', async () => { + const wrongFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`]; + for (const file of wrongFiles) { + assert.throws(() => extractMapSheetName(file), new Error('Version not found in the file name')); + } + }); +}); diff --git a/src/commands/basemaps-topo-import/list-jobs.ts b/src/commands/basemaps-topo-import/list-jobs.ts index 2bdedb05..64dfd741 100644 --- a/src/commands/basemaps-topo-import/list-jobs.ts +++ b/src/commands/basemaps-topo-import/list-jobs.ts @@ -1,27 +1,38 @@ import { fsa } from '@chunkd/fs'; import { command, option, string } from 'cmd-ts'; +import path from 'path'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; import { isArgo } from '../../utils/argo.js'; -import { config, forceOutput, registerCli, verbose } from '../common.js'; +import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; +import { isTiff } from '../tileindex-validate/tileindex.validate.js'; -type Output = { - output: string; +interface Output { + /** + * Input Topo Raster tiff files for processing + */ input: string[]; -}; + + /** + * Output name for the processed tiff files + */ + output: string; +} /** * List all the tiffs in a directory and output the list of names to process. * Outputs a json file with the list of files to process. * * @example - * [{ + * [ + * { * "output": "CJ10_v1-00", * "input": [ * "s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" - * ] - * }] + * ] + * } + * ] * * @param location: Location of the source files * @example s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ @@ -47,26 +58,36 @@ export const basemapsListTopoJobs = command({ const files = await fsa.toArray(fsa.list(args.location)); const outputs: Output[] = []; - for (const file of files) { logger.info({ file }, 'ListJobs:File'); + if (!isTiff(file)) continue; - if (file.endsWith('.tif') || file.endsWith('.tiff')) { - const fileUrl = new URL(file); - const tileName = fileUrl.pathname.split('/').pop(); - if (tileName == null) throw new Error(`Cannot get the tile name from file ${file}`); - - //Convert to the output filename - const output = tileName.replace('.tif', '').replace('GRIDLESS_GeoTif', ''); - logger.info({ output }, 'ListJobs:Output'); - outputs.push({ output, input: [file] }); - } + const output = extractMapSheetName(file); + outputs.push({ output, input: [file] }); } if (outputs.length === 0) throw new Error('No tiff files found in the location'); - if (args.forceOutput || isArgo()) + if (args.forceOutput || isArgo()) { await fsa.write('/tmp/list-topo-jobs/file-list.json', JSON.stringify(outputs, null, 2)); + } + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); }, }); + +export function extractMapSheetName(file: string): string { + const url = tryParseUrl(file); + const filePath = path.parse(url.href); + const tileName = filePath.name; + + // pull map sheet name off front of tileName (e.g. CJ10) + const mapSheetName = tileName.split('_')[0]; + // pull version off end of tileName (e.g. v1-0) + const version = tileName.match(/v(\d)-(\d\d)/)?.[0]; + if (version == null) throw new Error('Version not found in the file name'); + + const output = `${mapSheetName}_${version}`; + logger.info({ mapSheetName, version, output }, 'ListJobs:Output'); + return output; +} From b6a8d7a330b4d9e21b75e5ab61a2f2319d8248ed Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Fri, 29 Nov 2024 16:14:03 +1300 Subject: [PATCH 04/47] merge conficts --- src/commands/stac-setup/category.constants.ts | 10 ++++++++++ src/commands/stac-setup/stac.setup.ts | 1 + 2 files changed, 11 insertions(+) create mode 100644 src/commands/stac-setup/category.constants.ts diff --git a/src/commands/stac-setup/category.constants.ts b/src/commands/stac-setup/category.constants.ts new file mode 100644 index 00000000..1b6a6a81 --- /dev/null +++ b/src/commands/stac-setup/category.constants.ts @@ -0,0 +1,10 @@ +export const dataCategories = { + AERIAL_PHOTOS: 'aerial-photos', + SCANNED_AERIAL_PHOTOS: 'scanned-aerial-photos', + RURAL_AERIAL_PHOTOS: 'rural-aerial-photos', + SATELLITE_IMAGERY: 'satellite-imagery', + URBAN_AERIAL_PHOTOS: 'urban-aerial-photos', + TOPOGRAPHIC_MAPS: 'topographic-maps', + DEM: 'dem', + DSM: 'dsm', +}; diff --git a/src/commands/stac-setup/stac.setup.ts b/src/commands/stac-setup/stac.setup.ts index 88460752..2eea69f4 100644 --- a/src/commands/stac-setup/stac.setup.ts +++ b/src/commands/stac-setup/stac.setup.ts @@ -134,6 +134,7 @@ export function slugFromMetadata(metadata: SlugMetadata): string { ) { return `${slug}_${metadata.gsd}m`; } + if ( ([GeospatialDataCategories.Dem, GeospatialDataCategories.Dsm] as string[]).includes(metadata.geospatialCategory) ) { From 50ef9ca025a424d9f5123932554b833e7561f1c4 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 26 Nov 2024 13:55:43 +1300 Subject: [PATCH 05/47] Process standardising in the argo-tasks for topo map --- package-lock.json | 5212 +++++++++++++++-- package.json | 2 + .../__test__/list-jobs.test.ts | 30 - .../basemaps-topo-import/list-jobs.ts | 93 - .../__test__/import-topographic-maps.test.ts | 37 + .../basemaps-topo-map-import/gdal-commands.ts | 32 + .../import-topographic-maps.ts | 135 + src/commands/index.ts | 4 +- 8 files changed, 4923 insertions(+), 622 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/__test__/list-jobs.test.ts delete mode 100644 src/commands/basemaps-topo-import/list-jobs.ts create mode 100644 src/commands/basemaps-topo-map-import/__test__/import-topographic-maps.test.ts create mode 100644 src/commands/basemaps-topo-map-import/gdal-commands.ts create mode 100644 src/commands/basemaps-topo-map-import/import-topographic-maps.ts diff --git a/package-lock.json b/package-lock.json index 08ee5892..af911148 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,10 @@ "@aws-sdk/client-s3": "^3.440.0", "@aws-sdk/credential-providers": "^3.438.0", "@aws-sdk/lib-storage": "^3.440.0", + "@basemaps/cogify": "^7.12.0", "@basemaps/config": "^7.7.0", "@basemaps/geo": "^7.5.0", + "@basemaps/shared": "^7.12.0", "@chunkd/fs": "^10.0.9", "@chunkd/source-aws-v3": "^10.1.3", "@cogeotiff/core": "^9.0.3", @@ -60,19 +62,24 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/crc32c": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz", - "integrity": "sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-5.2.0.tgz", + "integrity": "sha512-+iWb8qaHLYKrNvGRbiYRHSdKRWhto5XlZUEBwDjYNf+ly5SVYG6zEoYIdxvf5R3zyeP16w4PLBn3rH1xc74Rag==", "dependencies": { - "@aws-crypto/util": "^3.0.0", + "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" + "tslib": "^2.6.2" } }, - "node_modules/@aws-crypto/crc32c/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + "node_modules/@aws-crypto/crc32c/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } }, "node_modules/@aws-crypto/ie11-detection": { "version": "3.0.0", @@ -88,23 +95,35 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" }, "node_modules/@aws-crypto/sha1-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz", - "integrity": "sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-5.2.0.tgz", + "integrity": "sha512-OH6lveCFfcDjX4dbAvCFSYUjJZjDr/3XJ3xHtjn3Oj5b9RjojQo8npoLeA/bNwkOkrSQ0wgrHzXk4tDRxGKJeg==", "dependencies": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", "@aws-sdk/types": "^3.222.0", "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" } }, - "node_modules/@aws-crypto/sha1-browser/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" + "node_modules/@aws-crypto/sha1-browser/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha1-browser/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } }, "node_modules/@aws-crypto/sha256-browser": { "version": "3.0.0", @@ -218,253 +237,2511 @@ "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/client-s3": { - "version": "3.440.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.440.0.tgz", - "integrity": "sha512-BIALZvf7BNfiY68YwNhQv2OPHER6q+AMTXrpKNXUAL5xJid86FqXiLDB0JmJXjZM4DOf+QUYuT6Ee60LVQgmaQ==", - "dependencies": { - "@aws-crypto/sha1-browser": "3.0.0", - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.438.0", - "@aws-sdk/core": "3.436.0", - "@aws-sdk/credential-provider-node": "3.438.0", - "@aws-sdk/middleware-bucket-endpoint": "3.433.0", - "@aws-sdk/middleware-expect-continue": "3.433.0", - "@aws-sdk/middleware-flexible-checksums": "3.433.0", - "@aws-sdk/middleware-host-header": "3.433.0", - "@aws-sdk/middleware-location-constraint": "3.433.0", - "@aws-sdk/middleware-logger": "3.433.0", - "@aws-sdk/middleware-recursion-detection": "3.433.0", - "@aws-sdk/middleware-sdk-s3": "3.440.0", - "@aws-sdk/middleware-signing": "3.433.0", - "@aws-sdk/middleware-ssec": "3.433.0", - "@aws-sdk/middleware-user-agent": "3.438.0", - "@aws-sdk/region-config-resolver": "3.433.0", - "@aws-sdk/signature-v4-multi-region": "3.437.0", - "@aws-sdk/types": "3.433.0", - "@aws-sdk/util-endpoints": "3.438.0", - "@aws-sdk/util-user-agent-browser": "3.433.0", - "@aws-sdk/util-user-agent-node": "3.437.0", - "@aws-sdk/xml-builder": "3.310.0", - "@smithy/config-resolver": "^2.0.16", - "@smithy/eventstream-serde-browser": "^2.0.12", - "@smithy/eventstream-serde-config-resolver": "^2.0.12", - "@smithy/eventstream-serde-node": "^2.0.12", - "@smithy/fetch-http-handler": "^2.2.4", - "@smithy/hash-blob-browser": "^2.0.12", - "@smithy/hash-node": "^2.0.12", - "@smithy/hash-stream-node": "^2.0.12", - "@smithy/invalid-dependency": "^2.0.12", - "@smithy/md5-js": "^2.0.12", - "@smithy/middleware-content-length": "^2.0.14", - "@smithy/middleware-endpoint": "^2.1.3", - "@smithy/middleware-retry": "^2.0.18", - "@smithy/middleware-serde": "^2.0.12", - "@smithy/middleware-stack": "^2.0.6", - "@smithy/node-config-provider": "^2.1.3", - "@smithy/node-http-handler": "^2.1.8", - "@smithy/protocol-http": "^3.0.8", - "@smithy/smithy-client": "^2.1.12", - "@smithy/types": "^2.4.0", - "@smithy/url-parser": "^2.0.12", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.16", - "@smithy/util-defaults-mode-node": "^2.0.21", - "@smithy/util-endpoints": "^1.0.2", - "@smithy/util-retry": "^2.0.5", - "@smithy/util-stream": "^2.0.17", - "@smithy/util-utf8": "^2.0.0", - "@smithy/util-waiter": "^2.0.12", - "fast-xml-parser": "4.2.5", - "tslib": "^2.5.0" + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.699.0.tgz", + "integrity": "sha512-npf2ZPUbFyyeWb0Fmgs/hGdjeecyUyldVU6okwM9DaaeOtlUmXA9e1vtrplgRJs3DLJdDJCGSTrBI+4w0MtgGg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/client-sts": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-endpoint-discovery": "3.696.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "@smithy/util-waiter": "^3.1.9", + "@types/uuid": "^9.0.1", + "tslib": "^2.6.2", + "uuid": "^9.0.1" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/client-sso": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.438.0.tgz", - "integrity": "sha512-L/xKq+K78PShLku8x5gM6lZDUp7LhFJ2ksKH7Vll+exSZq+QUaxuzjp4gqdzh6B0oIshv2jssQlUa0ScOmVRMg==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.436.0", - "@aws-sdk/middleware-host-header": "3.433.0", - "@aws-sdk/middleware-logger": "3.433.0", - "@aws-sdk/middleware-recursion-detection": "3.433.0", - "@aws-sdk/middleware-user-agent": "3.438.0", - "@aws-sdk/region-config-resolver": "3.433.0", - "@aws-sdk/types": "3.433.0", - "@aws-sdk/util-endpoints": "3.438.0", - "@aws-sdk/util-user-agent-browser": "3.433.0", - "@aws-sdk/util-user-agent-node": "3.437.0", - "@smithy/config-resolver": "^2.0.16", - "@smithy/fetch-http-handler": "^2.2.4", - "@smithy/hash-node": "^2.0.12", - "@smithy/invalid-dependency": "^2.0.12", - "@smithy/middleware-content-length": "^2.0.14", - "@smithy/middleware-endpoint": "^2.1.3", - "@smithy/middleware-retry": "^2.0.18", - "@smithy/middleware-serde": "^2.0.12", - "@smithy/middleware-stack": "^2.0.6", - "@smithy/node-config-provider": "^2.1.3", - "@smithy/node-http-handler": "^2.1.8", - "@smithy/protocol-http": "^3.0.8", - "@smithy/smithy-client": "^2.1.12", - "@smithy/types": "^2.4.0", - "@smithy/url-parser": "^2.0.12", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.16", - "@smithy/util-defaults-mode-node": "^2.0.21", - "@smithy/util-endpoints": "^1.0.2", - "@smithy/util-retry": "^2.0.5", + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" }, "engines": { "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/client-sts": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.438.0.tgz", - "integrity": "sha512-UBxLZKVVvbR4LHwSNSqaKx22YBSOGkavrh4SyDP8o8XOlXeRxTCllfSfjL9K5Mktp+ZwQ2NiubNcwmvUcGKbbg==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.436.0", - "@aws-sdk/credential-provider-node": "3.438.0", - "@aws-sdk/middleware-host-header": "3.433.0", - "@aws-sdk/middleware-logger": "3.433.0", - "@aws-sdk/middleware-recursion-detection": "3.433.0", - "@aws-sdk/middleware-sdk-sts": "3.433.0", - "@aws-sdk/middleware-signing": "3.433.0", - "@aws-sdk/middleware-user-agent": "3.438.0", - "@aws-sdk/region-config-resolver": "3.433.0", - "@aws-sdk/types": "3.433.0", - "@aws-sdk/util-endpoints": "3.438.0", - "@aws-sdk/util-user-agent-browser": "3.433.0", - "@aws-sdk/util-user-agent-node": "3.437.0", - "@smithy/config-resolver": "^2.0.16", - "@smithy/fetch-http-handler": "^2.2.4", - "@smithy/hash-node": "^2.0.12", - "@smithy/invalid-dependency": "^2.0.12", - "@smithy/middleware-content-length": "^2.0.14", - "@smithy/middleware-endpoint": "^2.1.3", - "@smithy/middleware-retry": "^2.0.18", - "@smithy/middleware-serde": "^2.0.12", - "@smithy/middleware-stack": "^2.0.6", - "@smithy/node-config-provider": "^2.1.3", - "@smithy/node-http-handler": "^2.1.8", - "@smithy/protocol-http": "^3.0.8", - "@smithy/smithy-client": "^2.1.12", - "@smithy/types": "^2.4.0", - "@smithy/url-parser": "^2.0.12", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.16", - "@smithy/util-defaults-mode-node": "^2.0.21", - "@smithy/util-endpoints": "^1.0.2", - "@smithy/util-retry": "^2.0.5", - "@smithy/util-utf8": "^2.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.5.0" + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/core": { - "version": "3.436.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.436.0.tgz", - "integrity": "sha512-vX5/LjXvCejC2XUY6TSg1oozjqK6BvkE75t0ys9dgqyr5PlZyZksMoeAFHUlj0sCjhT3ziWCujP1oiSpPWY9hg==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", "dependencies": { - "@smithy/smithy-client": "^2.1.12" + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.438.0.tgz", - "integrity": "sha512-/HgSPPvzIQ25SMII0vYlarJbijOAsXZCjayKWZ7+hilzju22hMB0ZTPM1E3QopWoZ6os76K59aAACfjhVAfIUg==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.438.0", - "@aws-sdk/types": "3.433.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.433.0.tgz", - "integrity": "sha512-Vl7Qz5qYyxBurMn6hfSiNJeUHSqfVUlMt0C1Bds3tCkl3IzecRWwyBOlxtxO3VCrgVeW3HqswLzCvhAFzPH6nQ==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" }, "engines": { "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.435.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.435.0.tgz", - "integrity": "sha512-i07YSy3+IrXwAzp3goCMo2OYzAwqRGIWPNMUX5ziFgA1eMlRWNC2slnbqJzax6xHrU8HdpNESAfflnQvUVBqYQ==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/fetch-http-handler": "^2.2.4", - "@smithy/node-http-handler": "^2.1.8", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/smithy-client": "^2.1.12", - "@smithy/types": "^2.4.0", - "@smithy/util-stream": "^2.0.17", - "tslib": "^2.5.0" + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.438.0.tgz", - "integrity": "sha512-WYPQR3pXoHJjn9/RMWipUhsUNFy6zhOiII6u8LJ5w84aNqIjV4+BdRYztRNGJD98jdtekhbkX0YKoSuZqP+unQ==", + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", "dependencies": { - "@aws-sdk/credential-provider-env": "3.433.0", - "@aws-sdk/credential-provider-process": "3.433.0", - "@aws-sdk/credential-provider-sso": "3.438.0", - "@aws-sdk/credential-provider-web-identity": "3.433.0", - "@aws-sdk/types": "3.433.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" }, "engines": { "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.438.0.tgz", - "integrity": "sha512-uaw3D2R0svyrC32qyZ2aOv/l0AT9eClh+eQsZJTQD3Kz9q+2VdeOBThQ8fsMfRtm26nUbZo6A/CRwxkm6okI+w==", - "dependencies": { + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sso": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.696.0.tgz", + "integrity": "sha512-q5TTkd08JS0DOkHfUL853tuArf7NrPeqoS5UOvqJho8ibV9Ak/a/HO4kNvy9Nj3cib/toHYHsQIEtecUPSUUrQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.699.0.tgz", + "integrity": "sha512-u8a1GorY5D1l+4FQAf4XBUC1T10/t7neuwT21r0ymrtMFSK2a9QqVHKMoLkvavAwyhJnARSBM9/UQC797PFOFw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/client-sts": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.699.0.tgz", + "integrity": "sha512-++lsn4x2YXsZPIzFVwv3fSUVM55ZT0WRFmPeNilYIhZClxHLmVAWKH4I55cY9ry60/aTKYjzOXkWwyBKGsGvQg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/core": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.696.0.tgz", + "integrity": "sha512-3c9III1k03DgvRZWg8vhVmfIXPG6hAciN9MzQTzqGngzWAELZF/WONRTRQuDFixVtarQatmLHYVw/atGeA2Byw==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.696.0.tgz", + "integrity": "sha512-T9iMFnJL7YTlESLpVFT3fg1Lkb1lD+oiaIC8KMpepb01gDUBIpj9+Y+pA/cgRWW0yRxmkDXNazAE2qQTVFGJzA==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.696.0.tgz", + "integrity": "sha512-GV6EbvPi2eq1+WgY/o2RFA3P7HGmnkIzCNmhwtALFlqMroLYWKE7PSeHw66Uh1dFQeVESn0/+hiUNhu1mB0emA==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.699.0.tgz", + "integrity": "sha512-dXmCqjJnKmG37Q+nLjPVu22mNkrGHY8hYoOt3Jo9R2zr5MYV7s/NHsCHr+7E+BZ+tfZYLRPeB1wkpTeHiEcdRw==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.699.0.tgz", + "integrity": "sha512-MmEmNDo1bBtTgRmdNfdQksXu4uXe66s0p1hi1YPrn1h59Q605eq/xiWbGL6/3KdkViH6eGUuABeV2ODld86ylg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-ini": "3.699.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.696.0.tgz", + "integrity": "sha512-mL1RcFDe9sfmyU5K1nuFkO8UiJXXxLX4JO1gVaDIOvPqwStpUAwi3A1BoeZhWZZNQsiKI810RnYGo0E0WB/hUA==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.699.0.tgz", + "integrity": "sha512-Ekp2cZG4pl9D8+uKWm4qO1xcm8/MeiI8f+dnlZm8aQzizeC+aXYy9GyoclSf6daK8KfRPiRfM7ZHBBL5dAfdMA==", + "dependencies": { + "@aws-sdk/client-sso": "3.696.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/token-providers": "3.699.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.696.0.tgz", + "integrity": "sha512-XJ/CVlWChM0VCoc259vWguFUjJDn/QwDqHwbx+K9cg3v6yrqXfK5ai+p/6lx0nQpnk4JzPVeYYxWRpaTsGC9rg==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.696.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.696.0.tgz", + "integrity": "sha512-zELJp9Ta2zkX7ELggMN9qMCgekqZhFC5V2rOr4hJDEb/Tte7gpfKSObAnw/3AYiVqt36sjHKfdkoTsuwGdEoDg==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-logger": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.696.0.tgz", + "integrity": "sha512-KhkHt+8AjCxcR/5Zp3++YPJPpFQzxpr+jmONiT/Jw2yqnSngZ0Yspm5wGoRx2hS1HJbyZNuaOWEGuJoxLeBKfA==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.696.0.tgz", + "integrity": "sha512-si/maV3Z0hH7qa99f9ru2xpS5HlfSVcasRlNUXKSDm611i7jFMWwGNLUOXFAOLhXotPX5G3Z6BLwL34oDeBMug==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.696.0.tgz", + "integrity": "sha512-Lvyj8CTyxrHI6GHd2YVZKIRI5Fmnugt3cpJo0VrKKEgK5zMySwEZ1n4dqPK6czYRWKd5+WnYHYAuU+Wdk6Jsjw==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.696.0.tgz", + "integrity": "sha512-7EuH142lBXjI8yH6dVS/CZeiK/WZsmb/8zP6bQbVYpMrppSTgB3MzZZdxVZGzL5r8zPQOU10wLC4kIMy0qdBVQ==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/token-providers": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.699.0.tgz", + "integrity": "sha512-kuiEW9DWs7fNos/SM+y58HCPhcIzm1nEZLhe2/7/6+TvAYLuEWURYsbK48gzsxXlaJ2k/jGY3nIsA7RptbMOwA==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-endpoints": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.696.0.tgz", + "integrity": "sha512-T5s0IlBVX+gkb9g/I6CLt4yAZVzMSiGnbUqWihWsHvQR1WOoIcndQy/Oz/IJXT9T2ipoy7a80gzV6a5mglrioA==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "@smithy/util-endpoints": "^2.1.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.696.0.tgz", + "integrity": "sha512-Z5rVNDdmPOe6ELoM5AhF/ja5tSjbe6ctSctDPb0JdDf4dT0v2MfwhJKzXju2RzX8Es/77Glh7MlaXLE0kCB9+Q==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.696.0.tgz", + "integrity": "sha512-KhKqcfyXIB0SCCt+qsu4eJjsfiOrNzK5dCV7RAW2YIpp+msxGUUX0NdRE9rkzjiv+3EMktgJm3eEIS+yxtlVdQ==", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/config-resolver": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.12.tgz", + "integrity": "sha512-YAJP9UJFZRZ8N+UruTeq78zkdjUHmzsY62J4qKWZ4SXB4QXJ/+680EfXXgkYA2xj77ooMqtUY9m406zGNqwivQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/credential-provider-imds": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.7.tgz", + "integrity": "sha512-cEfbau+rrWF8ylkmmVAObOmjbTIzKyUC5TkBL58SbLywD0RCBC4JAUKbmtSm2w5KUJNRPGgpGFMvE2FKnuNlWQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.10", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/fetch-http-handler": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.1.tgz", + "integrity": "sha512-bH7QW0+JdX0bPBadXt8GwMof/jz0H28I84hU1Uet9ISpzUqXqRQ3fEZJ+ANPOhzSEczYvANNl3uDQDYArSFDtA==", + "dependencies": { + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/hash-node": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.10.tgz", + "integrity": "sha512-3zWGWCHI+FlJ5WJwx73Mw2llYR8aflVyZN5JhoqLxbdPZi6UyKSdCeXAWJw9ja22m6S6Tzz1KZ+kAaSwvydi0g==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/invalid-dependency": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.10.tgz", + "integrity": "sha512-Lp2L65vFi+cj0vFMu2obpPW69DU+6O5g3086lmI4XcnRCG8PxvpWC7XyaVwJCxsZFzueHjXnrOH/E0pl0zikfA==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-content-length": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.12.tgz", + "integrity": "sha512-1mDEXqzM20yywaMDuf5o9ue8OkJ373lSPbaSjyEvkWdqELhFMyNNgKGWL/rCSf4KME8B+HlHKuR8u9kRj8HzEQ==", + "dependencies": { + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-endpoint": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.4.tgz", + "integrity": "sha512-TybiW2LA3kYVd3e+lWhINVu1o26KJbBwOpADnf0L4x/35vLVica77XVR5hvV9+kWeTGeSJ3IHTcYxbRxlbwhsg==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-retry": { + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.28.tgz", + "integrity": "sha512-vK2eDfvIXG1U64FEUhYxoZ1JSj4XFbYWkK36iz02i3pFwWiDz1Q7jKhGTBCwx/7KqJNk4VS7d7cDLXFOvP7M+g==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/service-error-classification": "^3.0.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-serde": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.10.tgz", + "integrity": "sha512-MnAuhh+dD14F428ubSJuRnmRsfOpxSzvRhaGVTvd/lrUDE3kxzCCmH8lnVTvoNQnV2BbJ4c15QwZ3UdQBtFNZA==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/middleware-stack": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.10.tgz", + "integrity": "sha512-grCHyoiARDBBGPyw2BeicpjgpsDFWZZxptbVKb3CRd/ZA15F/T6rZjCCuBUjJwdck1nwUuIxYtsS4H9DDpbP5w==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/node-config-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/node-http-handler": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/property-provider": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/querystring-builder": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/querystring-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.10.tgz", + "integrity": "sha512-Oa0XDcpo9SmjhiDD9ua2UyM3uU01ZTuIrNdZvzwUTykW1PM8o2yJvMh1Do1rY5sUQg4NDV70dMi0JhDx4GyxuQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/service-error-classification": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.10.tgz", + "integrity": "sha512-zHe642KCqDxXLuhs6xmHVgRwy078RfqxP2wRDpIyiF8EmsWXptMwnMwbVa50lw+WOGNrYm9zbaEg0oDe3PTtvQ==", + "dependencies": { + "@smithy/types": "^3.7.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/signature-v4": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.3.tgz", + "integrity": "sha512-pPSQQ2v2vu9vc8iew7sszLd0O09I5TRc5zhY71KA+Ao0xYazIG+uLeHbTJfIWGO3BGVLiXjUr3EEeCcEQLjpWQ==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/smithy-client": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.4.5.tgz", + "integrity": "sha512-k0sybYT9zlP79sIKd1XGm4TmK0AS1nA2bzDHXx7m0nGi3RQ8dxxQUs4CPkSmQTKAo+KF9aINU3KzpGIpV7UoMw==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-endpoint": "^3.2.4", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/url-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.10.tgz", + "integrity": "sha512-j90NUalTSBR2NaZTuruEgavSdh8MLirf58LoGSk4AtQfyIymogIhgnGUU2Mga2bkMkpSoC9gxb74xBXL5afKAQ==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.28.tgz", + "integrity": "sha512-6bzwAbZpHRFVJsOztmov5PGDmJYsbNSoIEfHSJJyFLzfBGCCChiO3od9k7E/TLgrCsIifdAbB9nqbVbyE7wRUw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.28.tgz", + "integrity": "sha512-78ENJDorV1CjOQselGmm3+z7Yqjj5HWCbjzh0Ixuq736dh1oEnD9sAttSBNSLlpZsX8VQnmERqA2fEFlmqWn8w==", + "dependencies": { + "@smithy/config-resolver": "^3.0.12", + "@smithy/credential-provider-imds": "^3.2.7", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-endpoints": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.1.6.tgz", + "integrity": "sha512-mFV1t3ndBh0yZOJgWxO9J/4cHZVn5UG1D8DeCc6/echfNkeEJWu9LD7mgGH5fHrEdR7LDoWw7PQO6QiGpHXhgA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-middleware": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-retry": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.10.tgz", + "integrity": "sha512-1l4qatFp4PiU6j7UsbasUHL2VU023NRB/gfaa1M0rDqVrRN4g3mCArLRyH3OuktApA4ye+yjWQHjdziunw2eWA==", + "dependencies": { + "@smithy/service-error-classification": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-stream": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.1.tgz", + "integrity": "sha512-Ff68R5lJh2zj+AUTvbAU/4yx+6QPRzg7+pI7M1FbtQHcRIp7xvguxVsQBKyB3fwiOwhAKu0lnNyYBaQfSW6TNw==", + "dependencies": { + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@aws-sdk/client-dynamodb/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@aws-sdk/client-s3": { + "version": "3.700.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.700.0.tgz", + "integrity": "sha512-TZTc8OZ873VodJNcsQ4Y/60f0Y0Ws5Z3Xm5QBgPCvhqQS7/+V28pXM3fAomJsKDWTxzH0nP9csX5cPvLybgdZQ==", + "dependencies": { + "@aws-crypto/sha1-browser": "5.2.0", + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/client-sts": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-bucket-endpoint": "3.696.0", + "@aws-sdk/middleware-expect-continue": "3.696.0", + "@aws-sdk/middleware-flexible-checksums": "3.697.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-location-constraint": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-sdk-s3": "3.696.0", + "@aws-sdk/middleware-ssec": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/signature-v4-multi-region": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@aws-sdk/xml-builder": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/eventstream-serde-browser": "^3.0.13", + "@smithy/eventstream-serde-config-resolver": "^3.0.10", + "@smithy/eventstream-serde-node": "^3.0.12", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-blob-browser": "^3.1.9", + "@smithy/hash-node": "^3.0.10", + "@smithy/hash-stream-node": "^3.1.9", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/md5-js": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-stream": "^3.3.1", + "@smithy/util-utf8": "^3.0.0", + "@smithy/util-waiter": "^3.1.9", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/sha256-browser/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sso": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.696.0.tgz", + "integrity": "sha512-q5TTkd08JS0DOkHfUL853tuArf7NrPeqoS5UOvqJho8ibV9Ak/a/HO4kNvy9Nj3cib/toHYHsQIEtecUPSUUrQ==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sso-oidc": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.699.0.tgz", + "integrity": "sha512-u8a1GorY5D1l+4FQAf4XBUC1T10/t7neuwT21r0ymrtMFSK2a9QqVHKMoLkvavAwyhJnARSBM9/UQC797PFOFw==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sts": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.699.0.tgz", + "integrity": "sha512-++lsn4x2YXsZPIzFVwv3fSUVM55ZT0WRFmPeNilYIhZClxHLmVAWKH4I55cY9ry60/aTKYjzOXkWwyBKGsGvQg==", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/client-sso-oidc": "3.699.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-node": "3.699.0", + "@aws-sdk/middleware-host-header": "3.696.0", + "@aws-sdk/middleware-logger": "3.696.0", + "@aws-sdk/middleware-recursion-detection": "3.696.0", + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/region-config-resolver": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@aws-sdk/util-user-agent-browser": "3.696.0", + "@aws-sdk/util-user-agent-node": "3.696.0", + "@smithy/config-resolver": "^3.0.12", + "@smithy/core": "^2.5.3", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/hash-node": "^3.0.10", + "@smithy/invalid-dependency": "^3.0.10", + "@smithy/middleware-content-length": "^3.0.12", + "@smithy/middleware-endpoint": "^3.2.3", + "@smithy/middleware-retry": "^3.0.27", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-body-length-node": "^3.0.0", + "@smithy/util-defaults-mode-browser": "^3.0.27", + "@smithy/util-defaults-mode-node": "^3.0.27", + "@smithy/util-endpoints": "^2.1.6", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/core": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.696.0.tgz", + "integrity": "sha512-3c9III1k03DgvRZWg8vhVmfIXPG6hAciN9MzQTzqGngzWAELZF/WONRTRQuDFixVtarQatmLHYVw/atGeA2Byw==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-env": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.696.0.tgz", + "integrity": "sha512-T9iMFnJL7YTlESLpVFT3fg1Lkb1lD+oiaIC8KMpepb01gDUBIpj9+Y+pA/cgRWW0yRxmkDXNazAE2qQTVFGJzA==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-http": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.696.0.tgz", + "integrity": "sha512-GV6EbvPi2eq1+WgY/o2RFA3P7HGmnkIzCNmhwtALFlqMroLYWKE7PSeHw66Uh1dFQeVESn0/+hiUNhu1mB0emA==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.699.0.tgz", + "integrity": "sha512-dXmCqjJnKmG37Q+nLjPVu22mNkrGHY8hYoOt3Jo9R2zr5MYV7s/NHsCHr+7E+BZ+tfZYLRPeB1wkpTeHiEcdRw==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-node": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.699.0.tgz", + "integrity": "sha512-MmEmNDo1bBtTgRmdNfdQksXu4uXe66s0p1hi1YPrn1h59Q605eq/xiWbGL6/3KdkViH6eGUuABeV2ODld86ylg==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.696.0", + "@aws-sdk/credential-provider-http": "3.696.0", + "@aws-sdk/credential-provider-ini": "3.699.0", + "@aws-sdk/credential-provider-process": "3.696.0", + "@aws-sdk/credential-provider-sso": "3.699.0", + "@aws-sdk/credential-provider-web-identity": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/credential-provider-imds": "^3.2.6", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-process": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.696.0.tgz", + "integrity": "sha512-mL1RcFDe9sfmyU5K1nuFkO8UiJXXxLX4JO1gVaDIOvPqwStpUAwi3A1BoeZhWZZNQsiKI810RnYGo0E0WB/hUA==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.699.0.tgz", + "integrity": "sha512-Ekp2cZG4pl9D8+uKWm4qO1xcm8/MeiI8f+dnlZm8aQzizeC+aXYy9GyoclSf6daK8KfRPiRfM7ZHBBL5dAfdMA==", + "dependencies": { + "@aws-sdk/client-sso": "3.696.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/token-providers": "3.699.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.696.0.tgz", + "integrity": "sha512-XJ/CVlWChM0VCoc259vWguFUjJDn/QwDqHwbx+K9cg3v6yrqXfK5ai+p/6lx0nQpnk4JzPVeYYxWRpaTsGC9rg==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sts": "^3.696.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-host-header": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.696.0.tgz", + "integrity": "sha512-zELJp9Ta2zkX7ELggMN9qMCgekqZhFC5V2rOr4hJDEb/Tte7gpfKSObAnw/3AYiVqt36sjHKfdkoTsuwGdEoDg==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-logger": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.696.0.tgz", + "integrity": "sha512-KhkHt+8AjCxcR/5Zp3++YPJPpFQzxpr+jmONiT/Jw2yqnSngZ0Yspm5wGoRx2hS1HJbyZNuaOWEGuJoxLeBKfA==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.696.0.tgz", + "integrity": "sha512-si/maV3Z0hH7qa99f9ru2xpS5HlfSVcasRlNUXKSDm611i7jFMWwGNLUOXFAOLhXotPX5G3Z6BLwL34oDeBMug==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-user-agent": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.696.0.tgz", + "integrity": "sha512-Lvyj8CTyxrHI6GHd2YVZKIRI5Fmnugt3cpJo0VrKKEgK5zMySwEZ1n4dqPK6czYRWKd5+WnYHYAuU+Wdk6Jsjw==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-endpoints": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/region-config-resolver": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.696.0.tgz", + "integrity": "sha512-7EuH142lBXjI8yH6dVS/CZeiK/WZsmb/8zP6bQbVYpMrppSTgB3MzZZdxVZGzL5r8zPQOU10wLC4kIMy0qdBVQ==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/token-providers": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.699.0.tgz", + "integrity": "sha512-kuiEW9DWs7fNos/SM+y58HCPhcIzm1nEZLhe2/7/6+TvAYLuEWURYsbK48gzsxXlaJ2k/jGY3nIsA7RptbMOwA==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/property-provider": "^3.1.9", + "@smithy/shared-ini-file-loader": "^3.1.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-sso-oidc": "^3.699.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-endpoints": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.696.0.tgz", + "integrity": "sha512-T5s0IlBVX+gkb9g/I6CLt4yAZVzMSiGnbUqWihWsHvQR1WOoIcndQy/Oz/IJXT9T2ipoy7a80gzV6a5mglrioA==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "@smithy/util-endpoints": "^2.1.6", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-user-agent-browser": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.696.0.tgz", + "integrity": "sha512-Z5rVNDdmPOe6ELoM5AhF/ja5tSjbe6ctSctDPb0JdDf4dT0v2MfwhJKzXju2RzX8Es/77Glh7MlaXLE0kCB9+Q==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-user-agent-node": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.696.0.tgz", + "integrity": "sha512-KhKqcfyXIB0SCCt+qsu4eJjsfiOrNzK5dCV7RAW2YIpp+msxGUUX0NdRE9rkzjiv+3EMktgJm3eEIS+yxtlVdQ==", + "dependencies": { + "@aws-sdk/middleware-user-agent": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/config-resolver": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.12.tgz", + "integrity": "sha512-YAJP9UJFZRZ8N+UruTeq78zkdjUHmzsY62J4qKWZ4SXB4QXJ/+680EfXXgkYA2xj77ooMqtUY9m406zGNqwivQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/credential-provider-imds": { + "version": "3.2.7", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.2.7.tgz", + "integrity": "sha512-cEfbau+rrWF8ylkmmVAObOmjbTIzKyUC5TkBL58SbLywD0RCBC4JAUKbmtSm2w5KUJNRPGgpGFMvE2FKnuNlWQ==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.10", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/fetch-http-handler": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.1.tgz", + "integrity": "sha512-bH7QW0+JdX0bPBadXt8GwMof/jz0H28I84hU1Uet9ISpzUqXqRQ3fEZJ+ANPOhzSEczYvANNl3uDQDYArSFDtA==", + "dependencies": { + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/hash-node": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.10.tgz", + "integrity": "sha512-3zWGWCHI+FlJ5WJwx73Mw2llYR8aflVyZN5JhoqLxbdPZi6UyKSdCeXAWJw9ja22m6S6Tzz1KZ+kAaSwvydi0g==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/invalid-dependency": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.10.tgz", + "integrity": "sha512-Lp2L65vFi+cj0vFMu2obpPW69DU+6O5g3086lmI4XcnRCG8PxvpWC7XyaVwJCxsZFzueHjXnrOH/E0pl0zikfA==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-content-length": { + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.12.tgz", + "integrity": "sha512-1mDEXqzM20yywaMDuf5o9ue8OkJ373lSPbaSjyEvkWdqELhFMyNNgKGWL/rCSf4KME8B+HlHKuR8u9kRj8HzEQ==", + "dependencies": { + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-endpoint": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.4.tgz", + "integrity": "sha512-TybiW2LA3kYVd3e+lWhINVu1o26KJbBwOpADnf0L4x/35vLVica77XVR5hvV9+kWeTGeSJ3IHTcYxbRxlbwhsg==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-retry": { + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.28.tgz", + "integrity": "sha512-vK2eDfvIXG1U64FEUhYxoZ1JSj4XFbYWkK36iz02i3pFwWiDz1Q7jKhGTBCwx/7KqJNk4VS7d7cDLXFOvP7M+g==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/service-error-classification": "^3.0.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-retry": "^3.0.10", + "tslib": "^2.6.2", + "uuid": "^9.0.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-serde": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.10.tgz", + "integrity": "sha512-MnAuhh+dD14F428ubSJuRnmRsfOpxSzvRhaGVTvd/lrUDE3kxzCCmH8lnVTvoNQnV2BbJ4c15QwZ3UdQBtFNZA==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-stack": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.10.tgz", + "integrity": "sha512-grCHyoiARDBBGPyw2BeicpjgpsDFWZZxptbVKb3CRd/ZA15F/T6rZjCCuBUjJwdck1nwUuIxYtsS4H9DDpbP5w==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-config-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-http-handler": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/property-provider": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/querystring-builder": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/querystring-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.10.tgz", + "integrity": "sha512-Oa0XDcpo9SmjhiDD9ua2UyM3uU01ZTuIrNdZvzwUTykW1PM8o2yJvMh1Do1rY5sUQg4NDV70dMi0JhDx4GyxuQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/service-error-classification": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.10.tgz", + "integrity": "sha512-zHe642KCqDxXLuhs6xmHVgRwy078RfqxP2wRDpIyiF8EmsWXptMwnMwbVa50lw+WOGNrYm9zbaEg0oDe3PTtvQ==", + "dependencies": { + "@smithy/types": "^3.7.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/signature-v4": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.3.tgz", + "integrity": "sha512-pPSQQ2v2vu9vc8iew7sszLd0O09I5TRc5zhY71KA+Ao0xYazIG+uLeHbTJfIWGO3BGVLiXjUr3EEeCcEQLjpWQ==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/smithy-client": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.4.5.tgz", + "integrity": "sha512-k0sybYT9zlP79sIKd1XGm4TmK0AS1nA2bzDHXx7m0nGi3RQ8dxxQUs4CPkSmQTKAo+KF9aINU3KzpGIpV7UoMw==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-endpoint": "^3.2.4", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/url-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.10.tgz", + "integrity": "sha512-j90NUalTSBR2NaZTuruEgavSdh8MLirf58LoGSk4AtQfyIymogIhgnGUU2Mga2bkMkpSoC9gxb74xBXL5afKAQ==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-body-length-node": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", + "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-defaults-mode-browser": { + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.28.tgz", + "integrity": "sha512-6bzwAbZpHRFVJsOztmov5PGDmJYsbNSoIEfHSJJyFLzfBGCCChiO3od9k7E/TLgrCsIifdAbB9nqbVbyE7wRUw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-defaults-mode-node": { + "version": "3.0.28", + "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.28.tgz", + "integrity": "sha512-78ENJDorV1CjOQselGmm3+z7Yqjj5HWCbjzh0Ixuq736dh1oEnD9sAttSBNSLlpZsX8VQnmERqA2fEFlmqWn8w==", + "dependencies": { + "@smithy/config-resolver": "^3.0.12", + "@smithy/credential-provider-imds": "^3.2.7", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.10", + "@smithy/smithy-client": "^3.4.5", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-endpoints": { + "version": "2.1.6", + "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.1.6.tgz", + "integrity": "sha512-mFV1t3ndBh0yZOJgWxO9J/4cHZVn5UG1D8DeCc6/echfNkeEJWu9LD7mgGH5fHrEdR7LDoWw7PQO6QiGpHXhgA==", + "dependencies": { + "@smithy/node-config-provider": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-middleware": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-retry": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.10.tgz", + "integrity": "sha512-1l4qatFp4PiU6j7UsbasUHL2VU023NRB/gfaa1M0rDqVrRN4g3mCArLRyH3OuktApA4ye+yjWQHjdziunw2eWA==", + "dependencies": { + "@smithy/service-error-classification": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-stream": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.1.tgz", + "integrity": "sha512-Ff68R5lJh2zj+AUTvbAU/4yx+6QPRzg7+pI7M1FbtQHcRIp7xvguxVsQBKyB3fwiOwhAKu0lnNyYBaQfSW6TNw==", + "dependencies": { + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, + "node_modules/@aws-sdk/client-s3/node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@aws-sdk/client-sso": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.438.0.tgz", + "integrity": "sha512-L/xKq+K78PShLku8x5gM6lZDUp7LhFJ2ksKH7Vll+exSZq+QUaxuzjp4gqdzh6B0oIshv2jssQlUa0ScOmVRMg==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/core": "3.436.0", + "@aws-sdk/middleware-host-header": "3.433.0", + "@aws-sdk/middleware-logger": "3.433.0", + "@aws-sdk/middleware-recursion-detection": "3.433.0", + "@aws-sdk/middleware-user-agent": "3.438.0", + "@aws-sdk/region-config-resolver": "3.433.0", + "@aws-sdk/types": "3.433.0", + "@aws-sdk/util-endpoints": "3.438.0", + "@aws-sdk/util-user-agent-browser": "3.433.0", + "@aws-sdk/util-user-agent-node": "3.437.0", + "@smithy/config-resolver": "^2.0.16", + "@smithy/fetch-http-handler": "^2.2.4", + "@smithy/hash-node": "^2.0.12", + "@smithy/invalid-dependency": "^2.0.12", + "@smithy/middleware-content-length": "^2.0.14", + "@smithy/middleware-endpoint": "^2.1.3", + "@smithy/middleware-retry": "^2.0.18", + "@smithy/middleware-serde": "^2.0.12", + "@smithy/middleware-stack": "^2.0.6", + "@smithy/node-config-provider": "^2.1.3", + "@smithy/node-http-handler": "^2.1.8", + "@smithy/protocol-http": "^3.0.8", + "@smithy/smithy-client": "^2.1.12", + "@smithy/types": "^2.4.0", + "@smithy/url-parser": "^2.0.12", + "@smithy/util-base64": "^2.0.0", + "@smithy/util-body-length-browser": "^2.0.0", + "@smithy/util-body-length-node": "^2.1.0", + "@smithy/util-defaults-mode-browser": "^2.0.16", + "@smithy/util-defaults-mode-node": "^2.0.21", + "@smithy/util-endpoints": "^1.0.2", + "@smithy/util-retry": "^2.0.5", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/client-sts": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.438.0.tgz", + "integrity": "sha512-UBxLZKVVvbR4LHwSNSqaKx22YBSOGkavrh4SyDP8o8XOlXeRxTCllfSfjL9K5Mktp+ZwQ2NiubNcwmvUcGKbbg==", + "dependencies": { + "@aws-crypto/sha256-browser": "3.0.0", + "@aws-crypto/sha256-js": "3.0.0", + "@aws-sdk/core": "3.436.0", + "@aws-sdk/credential-provider-node": "3.438.0", + "@aws-sdk/middleware-host-header": "3.433.0", + "@aws-sdk/middleware-logger": "3.433.0", + "@aws-sdk/middleware-recursion-detection": "3.433.0", + "@aws-sdk/middleware-sdk-sts": "3.433.0", + "@aws-sdk/middleware-signing": "3.433.0", + "@aws-sdk/middleware-user-agent": "3.438.0", + "@aws-sdk/region-config-resolver": "3.433.0", + "@aws-sdk/types": "3.433.0", + "@aws-sdk/util-endpoints": "3.438.0", + "@aws-sdk/util-user-agent-browser": "3.433.0", + "@aws-sdk/util-user-agent-node": "3.437.0", + "@smithy/config-resolver": "^2.0.16", + "@smithy/fetch-http-handler": "^2.2.4", + "@smithy/hash-node": "^2.0.12", + "@smithy/invalid-dependency": "^2.0.12", + "@smithy/middleware-content-length": "^2.0.14", + "@smithy/middleware-endpoint": "^2.1.3", + "@smithy/middleware-retry": "^2.0.18", + "@smithy/middleware-serde": "^2.0.12", + "@smithy/middleware-stack": "^2.0.6", + "@smithy/node-config-provider": "^2.1.3", + "@smithy/node-http-handler": "^2.1.8", + "@smithy/protocol-http": "^3.0.8", + "@smithy/smithy-client": "^2.1.12", + "@smithy/types": "^2.4.0", + "@smithy/url-parser": "^2.0.12", + "@smithy/util-base64": "^2.0.0", + "@smithy/util-body-length-browser": "^2.0.0", + "@smithy/util-body-length-node": "^2.1.0", + "@smithy/util-defaults-mode-browser": "^2.0.16", + "@smithy/util-defaults-mode-node": "^2.0.21", + "@smithy/util-endpoints": "^1.0.2", + "@smithy/util-retry": "^2.0.5", + "@smithy/util-utf8": "^2.0.0", + "fast-xml-parser": "4.2.5", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.436.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.436.0.tgz", + "integrity": "sha512-vX5/LjXvCejC2XUY6TSg1oozjqK6BvkE75t0ys9dgqyr5PlZyZksMoeAFHUlj0sCjhT3ziWCujP1oiSpPWY9hg==", + "dependencies": { + "@smithy/smithy-client": "^2.1.12" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-cognito-identity": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.438.0.tgz", + "integrity": "sha512-/HgSPPvzIQ25SMII0vYlarJbijOAsXZCjayKWZ7+hilzju22hMB0ZTPM1E3QopWoZ6os76K59aAACfjhVAfIUg==", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.438.0", + "@aws-sdk/types": "3.433.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.433.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.433.0.tgz", + "integrity": "sha512-Vl7Qz5qYyxBurMn6hfSiNJeUHSqfVUlMt0C1Bds3tCkl3IzecRWwyBOlxtxO3VCrgVeW3HqswLzCvhAFzPH6nQ==", + "dependencies": { + "@aws-sdk/types": "3.433.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.435.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.435.0.tgz", + "integrity": "sha512-i07YSy3+IrXwAzp3goCMo2OYzAwqRGIWPNMUX5ziFgA1eMlRWNC2slnbqJzax6xHrU8HdpNESAfflnQvUVBqYQ==", + "dependencies": { + "@aws-sdk/types": "3.433.0", + "@smithy/fetch-http-handler": "^2.2.4", + "@smithy/node-http-handler": "^2.1.8", + "@smithy/property-provider": "^2.0.0", + "@smithy/protocol-http": "^3.0.8", + "@smithy/smithy-client": "^2.1.12", + "@smithy/types": "^2.4.0", + "@smithy/util-stream": "^2.0.17", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.438.0.tgz", + "integrity": "sha512-WYPQR3pXoHJjn9/RMWipUhsUNFy6zhOiII6u8LJ5w84aNqIjV4+BdRYztRNGJD98jdtekhbkX0YKoSuZqP+unQ==", + "dependencies": { + "@aws-sdk/credential-provider-env": "3.433.0", + "@aws-sdk/credential-provider-process": "3.433.0", + "@aws-sdk/credential-provider-sso": "3.438.0", + "@aws-sdk/credential-provider-web-identity": "3.433.0", + "@aws-sdk/types": "3.433.0", + "@smithy/credential-provider-imds": "^2.0.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.438.0.tgz", + "integrity": "sha512-uaw3D2R0svyrC32qyZ2aOv/l0AT9eClh+eQsZJTQD3Kz9q+2VdeOBThQ8fsMfRtm26nUbZo6A/CRwxkm6okI+w==", + "dependencies": { "@aws-sdk/credential-provider-env": "3.433.0", "@aws-sdk/credential-provider-ini": "3.438.0", "@aws-sdk/credential-provider-process": "3.433.0", @@ -477,18 +2754,807 @@ "@smithy/types": "^2.4.0", "tslib": "^2.5.0" }, - "engines": { - "node": ">=14.0.0" + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.433.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.433.0.tgz", + "integrity": "sha512-W7FcGlQjio9Y/PepcZGRyl5Bpwb0uWU7qIUCh+u4+q2mW4D5ZngXg8V/opL9/I/p4tUH9VXZLyLGwyBSkdhL+A==", + "dependencies": { + "@aws-sdk/types": "3.433.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.438.0.tgz", + "integrity": "sha512-Xykli/64xR18cBV5P0XFxcH120omtfAjC/cFy/9nFU/+dPvbk0uu1yEOZYteWHyGGkPN4PkHmbh60GiUCLQkWQ==", + "dependencies": { + "@aws-sdk/client-sso": "3.438.0", + "@aws-sdk/token-providers": "3.438.0", + "@aws-sdk/types": "3.433.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.433.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.433.0.tgz", + "integrity": "sha512-RlwjP1I5wO+aPpwyCp23Mk8nmRbRL33hqRASy73c4JA2z2YiRua+ryt6MalIxehhwQU6xvXUKulJnPG9VaMFZg==", + "dependencies": { + "@aws-sdk/types": "3.433.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/credential-providers": { + "version": "3.438.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.438.0.tgz", + "integrity": "sha512-EBtcczPtUyXsN/yNGvZxGU/Ildl8kJeq7Vt7MsFLtOmYXDWoMsSIEVuSYbBdzBal1z03fmd/Mmjr0DhYiSAqMg==", + "dependencies": { + "@aws-sdk/client-cognito-identity": "3.438.0", + "@aws-sdk/client-sso": "3.438.0", + "@aws-sdk/client-sts": "3.438.0", + "@aws-sdk/credential-provider-cognito-identity": "3.438.0", + "@aws-sdk/credential-provider-env": "3.433.0", + "@aws-sdk/credential-provider-http": "3.435.0", + "@aws-sdk/credential-provider-ini": "3.438.0", + "@aws-sdk/credential-provider-node": "3.438.0", + "@aws-sdk/credential-provider-process": "3.433.0", + "@aws-sdk/credential-provider-sso": "3.438.0", + "@aws-sdk/credential-provider-web-identity": "3.433.0", + "@aws-sdk/types": "3.433.0", + "@smithy/credential-provider-imds": "^2.0.0", + "@smithy/property-provider": "^2.0.0", + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.693.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.693.0.tgz", + "integrity": "sha512-/zK0ZZncBf5FbTfo8rJMcQIXXk4Ibhe5zEMiwFNivVPR2uNC0+oqfwXz7vjxwY0t6BPE3Bs4h9uFEz4xuGCY6w==", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/lib-storage": { + "version": "3.440.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.440.0.tgz", + "integrity": "sha512-JZ1+vi9WwYWftvlkWvoNUbuhWRU8kN9YSjkQPqOb54hLqCY3les2KQqX1kwk1O6B3aQaw1+57uPqNSBeQDourw==", + "dependencies": { + "@smithy/abort-controller": "^2.0.1", + "@smithy/middleware-endpoint": "^2.1.3", + "@smithy/smithy-client": "^2.1.12", + "buffer": "5.6.0", + "events": "3.3.0", + "stream-browserify": "3.0.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.696.0.tgz", + "integrity": "sha512-V07jishKHUS5heRNGFpCWCSTjRJyQLynS/ncUeE8ZYtG66StOOQWftTwDfFOSoXlIqrXgb4oT9atryzXq7Z4LQ==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-arn-parser": "3.693.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/node-config-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/property-provider": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.696.0.tgz", + "integrity": "sha512-KZvgR3lB9zdLuuO+SxeQQVDn8R46Brlolsbv7JGyR6id0BNy6pqitHdcrZCyp9jaMjrSFcPROceeLy70Cu3pZg==", + "dependencies": { + "@aws-sdk/endpoint-cache": "3.693.0", + "@aws-sdk/types": "3.696.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/node-config-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/property-provider": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.696.0.tgz", + "integrity": "sha512-vpVukqY3U2pb+ULeX0shs6L0aadNep6kKzjme/MyulPjtUDJpD3AekHsXRrCCGLmOqSKqRgQn5zhV9pQhHsb6Q==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums": { + "version": "3.697.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.697.0.tgz", + "integrity": "sha512-K/y43P+NuHu5+21/29BoJSltcPekvcCU8i74KlGGHbW2Z105e5QVZlFjxivcPOjOA3gdC0W4SoFSIWam5RBhzw==", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@aws-crypto/crc32c": "5.2.0", + "@aws-crypto/util": "5.2.0", + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-stream": "^3.3.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-crypto/util/node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-crypto/util/node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-crypto/util/node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-sdk/core": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.696.0.tgz", + "integrity": "sha512-3c9III1k03DgvRZWg8vhVmfIXPG6hAciN9MzQTzqGngzWAELZF/WONRTRQuDFixVtarQatmLHYVw/atGeA2Byw==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/fetch-http-handler": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.1.tgz", + "integrity": "sha512-bH7QW0+JdX0bPBadXt8GwMof/jz0H28I84hU1Uet9ISpzUqXqRQ3fEZJ+ANPOhzSEczYvANNl3uDQDYArSFDtA==", + "dependencies": { + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/middleware-endpoint": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.4.tgz", + "integrity": "sha512-TybiW2LA3kYVd3e+lWhINVu1o26KJbBwOpADnf0L4x/35vLVica77XVR5hvV9+kWeTGeSJ3IHTcYxbRxlbwhsg==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/middleware-serde": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.10.tgz", + "integrity": "sha512-MnAuhh+dD14F428ubSJuRnmRsfOpxSzvRhaGVTvd/lrUDE3kxzCCmH8lnVTvoNQnV2BbJ4c15QwZ3UdQBtFNZA==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/middleware-stack": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.10.tgz", + "integrity": "sha512-grCHyoiARDBBGPyw2BeicpjgpsDFWZZxptbVKb3CRd/ZA15F/T6rZjCCuBUjJwdck1nwUuIxYtsS4H9DDpbP5w==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/node-config-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/node-http-handler": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/property-provider": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/querystring-builder": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/querystring-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.10.tgz", + "integrity": "sha512-Oa0XDcpo9SmjhiDD9ua2UyM3uU01ZTuIrNdZvzwUTykW1PM8o2yJvMh1Do1rY5sUQg4NDV70dMi0JhDx4GyxuQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/signature-v4": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.3.tgz", + "integrity": "sha512-pPSQQ2v2vu9vc8iew7sszLd0O09I5TRc5zhY71KA+Ao0xYazIG+uLeHbTJfIWGO3BGVLiXjUr3EEeCcEQLjpWQ==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/smithy-client": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.4.5.tgz", + "integrity": "sha512-k0sybYT9zlP79sIKd1XGm4TmK0AS1nA2bzDHXx7m0nGi3RQ8dxxQUs4CPkSmQTKAo+KF9aINU3KzpGIpV7UoMw==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-endpoint": "^3.2.4", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/url-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.10.tgz", + "integrity": "sha512-j90NUalTSBR2NaZTuruEgavSdh8MLirf58LoGSk4AtQfyIymogIhgnGUU2Mga2bkMkpSoC9gxb74xBXL5afKAQ==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-middleware": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-stream": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.1.tgz", + "integrity": "sha512-Ff68R5lJh2zj+AUTvbAU/4yx+6QPRzg7+pI7M1FbtQHcRIp7xvguxVsQBKyB3fwiOwhAKu0lnNyYBaQfSW6TNw==", + "dependencies": { + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" } }, - "node_modules/@aws-sdk/credential-provider-process": { + "node_modules/@aws-sdk/middleware-host-header": { "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.433.0.tgz", - "integrity": "sha512-W7FcGlQjio9Y/PepcZGRyl5Bpwb0uWU7qIUCh+u4+q2mW4D5ZngXg8V/opL9/I/p4tUH9VXZLyLGwyBSkdhL+A==", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.433.0.tgz", + "integrity": "sha512-mBTq3UWv1UzeHG+OfUQ2MB/5GEkt5LTKFaUqzL7ESwzW8XtpBgXnjZvIwu3Vcd3sEetMwijwaGiJhY0ae/YyaA==", "dependencies": { "@aws-sdk/types": "3.433.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", + "@smithy/protocol-http": "^3.0.8", "@smithy/types": "^2.4.0", "tslib": "^2.5.0" }, @@ -496,30 +3562,48 @@ "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.438.0.tgz", - "integrity": "sha512-Xykli/64xR18cBV5P0XFxcH120omtfAjC/cFy/9nFU/+dPvbk0uu1yEOZYteWHyGGkPN4PkHmbh60GiUCLQkWQ==", + "node_modules/@aws-sdk/middleware-location-constraint": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.696.0.tgz", + "integrity": "sha512-FgH12OB0q+DtTrP2aiDBddDKwL4BPOrm7w3VV9BJrSdkqQCNBPz8S1lb0y5eVH4tBG+2j7gKPlOv1wde4jF/iw==", "dependencies": { - "@aws-sdk/client-sso": "3.438.0", - "@aws-sdk/token-providers": "3.438.0", - "@aws-sdk/types": "3.433.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/credential-provider-web-identity": { + "node_modules/@aws-sdk/middleware-location-constraint/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-location-constraint/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-logger": { "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.433.0.tgz", - "integrity": "sha512-RlwjP1I5wO+aPpwyCp23Mk8nmRbRL33hqRASy73c4JA2z2YiRua+ryt6MalIxehhwQU6xvXUKulJnPG9VaMFZg==", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.433.0.tgz", + "integrity": "sha512-We346Fb5xGonTGVZC9Nvqtnqy74VJzYuTLLiuuftA5sbNzftBDy/22QCfvYSTOAl3bvif+dkDUzQY2ihc5PwOQ==", "dependencies": { "@aws-sdk/types": "3.433.0", - "@smithy/property-provider": "^2.0.0", "@smithy/types": "^2.4.0", "tslib": "^2.5.0" }, @@ -527,25 +3611,13 @@ "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/credential-providers": { - "version": "3.438.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.438.0.tgz", - "integrity": "sha512-EBtcczPtUyXsN/yNGvZxGU/Ildl8kJeq7Vt7MsFLtOmYXDWoMsSIEVuSYbBdzBal1z03fmd/Mmjr0DhYiSAqMg==", + "node_modules/@aws-sdk/middleware-recursion-detection": { + "version": "3.433.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.433.0.tgz", + "integrity": "sha512-HEvYC9PQlWY/ccUYtLvAlwwf1iCif2TSAmLNr3YTBRVa98x6jKL0hlCrHWYklFeqOGSKy6XhE+NGJMUII0/HaQ==", "dependencies": { - "@aws-sdk/client-cognito-identity": "3.438.0", - "@aws-sdk/client-sso": "3.438.0", - "@aws-sdk/client-sts": "3.438.0", - "@aws-sdk/credential-provider-cognito-identity": "3.438.0", - "@aws-sdk/credential-provider-env": "3.433.0", - "@aws-sdk/credential-provider-http": "3.435.0", - "@aws-sdk/credential-provider-ini": "3.438.0", - "@aws-sdk/credential-provider-node": "3.438.0", - "@aws-sdk/credential-provider-process": "3.433.0", - "@aws-sdk/credential-provider-sso": "3.438.0", - "@aws-sdk/credential-provider-web-identity": "3.433.0", "@aws-sdk/types": "3.433.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", + "@smithy/protocol-http": "^3.0.8", "@smithy/types": "^2.4.0", "tslib": "^2.5.0" }, @@ -553,143 +3625,405 @@ "node": ">=14.0.0" } }, - "node_modules/@aws-sdk/lib-storage": { - "version": "3.440.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.440.0.tgz", - "integrity": "sha512-JZ1+vi9WwYWftvlkWvoNUbuhWRU8kN9YSjkQPqOb54hLqCY3les2KQqX1kwk1O6B3aQaw1+57uPqNSBeQDourw==", + "node_modules/@aws-sdk/middleware-sdk-s3": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.696.0.tgz", + "integrity": "sha512-M7fEiAiN7DBMHflzOFzh1I2MNSlLpbiH2ubs87bdRc2wZsDPSbs4l3v6h3WLhxoQK0bq6vcfroudrLBgvCuX3Q==", + "dependencies": { + "@aws-sdk/core": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@aws-sdk/util-arn-parser": "3.693.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-config-provider": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-stream": "^3.3.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@aws-sdk/core": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.696.0.tgz", + "integrity": "sha512-3c9III1k03DgvRZWg8vhVmfIXPG6hAciN9MzQTzqGngzWAELZF/WONRTRQuDFixVtarQatmLHYVw/atGeA2Byw==", + "dependencies": { + "@aws-sdk/types": "3.696.0", + "@smithy/core": "^2.5.3", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/property-provider": "^3.1.9", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/smithy-client": "^3.4.4", + "@smithy/types": "^3.7.1", + "@smithy/util-middleware": "^3.0.10", + "fast-xml-parser": "4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", "dependencies": { - "@smithy/abort-controller": "^2.0.1", - "@smithy/middleware-endpoint": "^2.1.3", - "@smithy/smithy-client": "^2.1.12", - "buffer": "5.6.0", - "events": "3.3.0", - "stream-browserify": "3.0.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, - "peerDependencies": { - "@aws-sdk/client-s3": "^3.0.0" + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.433.0.tgz", - "integrity": "sha512-Lk1xIu2tWTRa1zDw5hCF1RrpWQYSodUhrS/q3oKz8IAoFqEy+lNaD5jx+fycuZb5EkE4IzWysT+8wVkd0mAnOg==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/fetch-http-handler": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.1.tgz", + "integrity": "sha512-bH7QW0+JdX0bPBadXt8GwMof/jz0H28I84hU1Uet9ISpzUqXqRQ3fEZJ+ANPOhzSEczYvANNl3uDQDYArSFDtA==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@aws-sdk/util-arn-parser": "3.310.0", - "@smithy/node-config-provider": "^2.1.3", - "@smithy/protocol-http": "^3.0.8", - "@smithy/types": "^2.4.0", - "@smithy/util-config-provider": "^2.0.0", - "tslib": "^2.5.0" + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.433.0.tgz", - "integrity": "sha512-Uq2rPIsjz0CR2sulM/HyYr5WiqiefrSRLdwUZuA7opxFSfE808w5DBWSprHxbH3rbDSQR4nFiOiVYIH8Eth7nA==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-endpoint": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.2.4.tgz", + "integrity": "sha512-TybiW2LA3kYVd3e+lWhINVu1o26KJbBwOpADnf0L4x/35vLVica77XVR5hvV9+kWeTGeSJ3IHTcYxbRxlbwhsg==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/core": "^2.5.4", + "@smithy/middleware-serde": "^3.0.10", + "@smithy/node-config-provider": "^3.1.11", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "@smithy/url-parser": "^3.0.10", + "@smithy/util-middleware": "^3.0.10", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.433.0.tgz", - "integrity": "sha512-Ptssx373+I7EzFUWjp/i/YiNFt6I6sDuRHz6DOUR9nmmRTlHHqmdcBXlJL2d9wwFxoBRCN8/PXGsTc/DJ4c95Q==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-serde": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.10.tgz", + "integrity": "sha512-MnAuhh+dD14F428ubSJuRnmRsfOpxSzvRhaGVTvd/lrUDE3kxzCCmH8lnVTvoNQnV2BbJ4c15QwZ3UdQBtFNZA==", "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@aws-crypto/crc32c": "3.0.0", - "@aws-sdk/types": "3.433.0", - "@smithy/is-array-buffer": "^2.0.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/types": "^2.4.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-stack": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.10.tgz", + "integrity": "sha512-grCHyoiARDBBGPyw2BeicpjgpsDFWZZxptbVKb3CRd/ZA15F/T6rZjCCuBUjJwdck1nwUuIxYtsS4H9DDpbP5w==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-config-provider": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.11.tgz", + "integrity": "sha512-URq3gT3RpDikh/8MBJUB+QGZzfS7Bm6TQTqoh4CqE8NBuyPkWa5eUXj0XFcFfeZVgg3WMh1u19iaXn8FvvXxZw==", + "dependencies": { + "@smithy/property-provider": "^3.1.10", + "@smithy/shared-ini-file-loader": "^3.1.11", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-http-handler": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/property-provider": { + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.10.tgz", + "integrity": "sha512-n1MJZGTorTH2DvyTVj+3wXnd4CzjJxyXeOgnTlgNVFxaaMeT4OteEp4QrzF8p9ee2yg42nvyVK6R/awLCakjeQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/querystring-builder": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/querystring-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.10.tgz", + "integrity": "sha512-Oa0XDcpo9SmjhiDD9ua2UyM3uU01ZTuIrNdZvzwUTykW1PM8o2yJvMh1Do1rY5sUQg4NDV70dMi0JhDx4GyxuQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/shared-ini-file-loader": { + "version": "3.1.11", + "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.11.tgz", + "integrity": "sha512-AUdrIZHFtUgmfSN4Gq9nHu3IkHMa1YDcN+s061Nfm+6pQ0mJy85YQDB0tZBCmls0Vuj22pLwDPmL92+Hvfwwlg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/signature-v4": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.3.tgz", + "integrity": "sha512-pPSQQ2v2vu9vc8iew7sszLd0O09I5TRc5zhY71KA+Ao0xYazIG+uLeHbTJfIWGO3BGVLiXjUr3EEeCcEQLjpWQ==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/smithy-client": { + "version": "3.4.5", + "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.4.5.tgz", + "integrity": "sha512-k0sybYT9zlP79sIKd1XGm4TmK0AS1nA2bzDHXx7m0nGi3RQ8dxxQUs4CPkSmQTKAo+KF9aINU3KzpGIpV7UoMw==", + "dependencies": { + "@smithy/core": "^2.5.4", + "@smithy/middleware-endpoint": "^3.2.4", + "@smithy/middleware-stack": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-stream": "^3.3.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/url-parser": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.10.tgz", + "integrity": "sha512-j90NUalTSBR2NaZTuruEgavSdh8MLirf58LoGSk4AtQfyIymogIhgnGUU2Mga2bkMkpSoC9gxb74xBXL5afKAQ==", + "dependencies": { + "@smithy/querystring-parser": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-config-provider": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", + "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", + "dependencies": { + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.433.0.tgz", - "integrity": "sha512-mBTq3UWv1UzeHG+OfUQ2MB/5GEkt5LTKFaUqzL7ESwzW8XtpBgXnjZvIwu3Vcd3sEetMwijwaGiJhY0ae/YyaA==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.433.0.tgz", - "integrity": "sha512-2YD860TGntwZifIUbxm+lFnNJJhByR/RB/+fV1I8oGKg+XX2rZU+94pRfHXRywoZKlCA0L+LGDA1I56jxrB9sw==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-middleware": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-logger": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.433.0.tgz", - "integrity": "sha512-We346Fb5xGonTGVZC9Nvqtnqy74VJzYuTLLiuuftA5sbNzftBDy/22QCfvYSTOAl3bvif+dkDUzQY2ihc5PwOQ==", - "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-stream": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.1.tgz", + "integrity": "sha512-Ff68R5lJh2zj+AUTvbAU/4yx+6QPRzg7+pI7M1FbtQHcRIp7xvguxVsQBKyB3fwiOwhAKu0lnNyYBaQfSW6TNw==", + "dependencies": { + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.433.0.tgz", - "integrity": "sha512-HEvYC9PQlWY/ccUYtLvAlwwf1iCif2TSAmLNr3YTBRVa98x6jKL0hlCrHWYklFeqOGSKy6XhE+NGJMUII0/HaQ==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.440.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.440.0.tgz", - "integrity": "sha512-DVTSr+82Z8jR9xTwDN3YHzxX7qvi0n96V92OfxvSRDq2BldCEx/KEL1orUZjw97SAXhINOlUWjRR7j4HpwWQtQ==", + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@aws-sdk/util-arn-parser": "3.310.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/smithy-client": "^2.1.12", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/fast-xml-parser": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.1.tgz", + "integrity": "sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" } }, "node_modules/@aws-sdk/middleware-sdk-sts": { @@ -724,16 +4058,39 @@ } }, "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.433.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.433.0.tgz", - "integrity": "sha512-2AMaPx0kYfCiekxoL7aqFqSSoA9du+yI4zefpQNLr+1cZOerYiDxdsZ4mbqStR1CVFaX6U6hrYokXzjInsvETw==", + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.696.0.tgz", + "integrity": "sha512-w/d6O7AOZ7Pg3w2d3BxnX5RmGNWb5X4RNxF19rJqcgu/xqxxE/QwZTNd5a7eTsqLXAUIfbbR8hh0czVfC1pJLA==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@aws-sdk/types": "3.696.0", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-ssec/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/middleware-ssec/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@aws-sdk/middleware-user-agent": { @@ -767,18 +4124,141 @@ } }, "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.437.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.437.0.tgz", - "integrity": "sha512-MmrqudssOs87JgVg7HGVdvJws/t4kcOrJJd+975ki+DPeSoyK2U4zBDfDkJ+n0tFuZBs3sLwLh0QXE7BV28rRA==", + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.696.0.tgz", + "integrity": "sha512-ijPkoLjXuPtgxAYlDoYls8UaG/VKigROn9ebbvPL/orEY5umedd3iZTcS9T+uAf4Ur3GELLxMQiERZpfDKaz3g==", + "dependencies": { + "@aws-sdk/middleware-sdk-s3": "3.696.0", + "@aws-sdk/types": "3.696.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/signature-v4": "^4.2.2", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@aws-sdk/types": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.696.0.tgz", + "integrity": "sha512-9rTvUJIAj5d3//U5FDPWGJ1nFJLuWb30vugGOrWk7aNZ6y9tuA3PI7Cc9dP8WEXKVyK1vuuk8rSFP2iqXnlgrw==", "dependencies": { - "@aws-sdk/types": "3.433.0", - "@smithy/protocol-http": "^3.0.8", - "@smithy/signature-v4": "^2.0.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/signature-v4": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-4.2.3.tgz", + "integrity": "sha512-pPSQQ2v2vu9vc8iew7sszLd0O09I5TRc5zhY71KA+Ao0xYazIG+uLeHbTJfIWGO3BGVLiXjUr3EEeCcEQLjpWQ==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-uri-escape": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-middleware": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@aws-sdk/token-providers": { @@ -841,14 +4321,28 @@ } }, "node_modules/@aws-sdk/util-arn-parser": { - "version": "3.310.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.310.0.tgz", - "integrity": "sha512-jL8509owp/xB9+Or0pvn3Fe+b94qfklc2yPowZZIFAkFcCSIdkIglz18cPDWnYAcy9JGewpMS1COXKIUhZkJsA==", + "version": "3.693.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.693.0.tgz", + "integrity": "sha512-WC8x6ca+NRrtpAH64rWu+ryDZI3HuLwlEr8EU6/dbC/pt+r/zC0PBoC15VEygUaBA+isppCikQpGyEDu0Yj7gQ==", "dependencies": { - "tslib": "^2.5.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/util-dynamodb": { + "version": "3.699.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.699.0.tgz", + "integrity": "sha512-OamtYYyvVuEXiGUmjrKQqPhAUD1L8I1Eb7XcLcKIJKjegtadxr8Alp+wiefz/4Tandmb5erOjNJNYUCU3KfPog==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.699.0" } }, "node_modules/@aws-sdk/util-endpoints": { @@ -897,56 +4391,151 @@ "tslib": "^2.5.0" }, "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } + "node": ">=14.0.0" + }, + "peerDependencies": { + "aws-crt": ">=1.0.0" + }, + "peerDependenciesMeta": { + "aws-crt": { + "optional": true + } + } + }, + "node_modules/@aws-sdk/util-utf8-browser": { + "version": "3.259.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", + "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", + "dependencies": { + "tslib": "^2.3.1" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.696.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.696.0.tgz", + "integrity": "sha512-dn1mX+EeqivoLYnY7p2qLrir0waPnCgS/0YdRCAVU2x14FgfUYCH6Im3w3oi2dMwhxfKY5lYVB5NKvZu7uI9lQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@basemaps/cogify": { + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/@basemaps/cogify/-/cogify-7.12.0.tgz", + "integrity": "sha512-mkTKqWcp/2FoDANFLDyDCd7dFmN3OalfEoQ2QzpP5Z2FhYcWj9RXKc75aElm2fBTHELFKmvskT0B3rDIN30tTg==", + "bin": { + "cogify": "build/bin.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/@basemaps/config": { + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/@basemaps/config/-/config-7.12.0.tgz", + "integrity": "sha512-Fm+rTYLNoh44Lu7nobCbEJcBE5EfYaZ+tappM+JbhzbSfyS3+dc02Hl5G2ZYfnFTJEfePDuGzS4C9XoJfjBD5w==", + "dependencies": { + "@basemaps/geo": "^7.12.0", + "base-x": "^4.0.0", + "zod": "^3.17.3" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@basemaps/geo": { + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/@basemaps/geo/-/geo-7.12.0.tgz", + "integrity": "sha512-t14ftk19u61AAjBQ7fU674E5vLlWquvqGtm+9q3lDaAca2tZSO1MYKA6pp8K1BmGnMxjn9mUQXCLeL6WCrAkbg==", + "dependencies": { + "@linzjs/tile-matrix-set": "^0.0.1", + "proj4": "^2.8.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@basemaps/shared": { + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/@basemaps/shared/-/shared-7.12.0.tgz", + "integrity": "sha512-ScK/mHcmkp2FnEYnD4rvanMcB6l056PJpJvkDVz9d1VIoV8dN1UXHL8fmCfOwliQ0OQ1vwtrv3MbriDTotrjbg==", + "dependencies": { + "@aws-sdk/client-dynamodb": "^3.470.0", + "@aws-sdk/client-s3": "^3.472.0", + "@aws-sdk/util-dynamodb": "^3.470.0", + "@basemaps/config": "^7.12.0", + "@basemaps/geo": "^7.12.0", + "@chunkd/fs": "^11.2.0", + "@chunkd/fs-aws": "^11.3.0", + "@chunkd/middleware": "^11.1.0", + "@cogeotiff/core": "^9.0.3", + "@cotar/core": "^6.0.1", + "@linzjs/geojson": "^7.10.0", + "@linzjs/metrics": "^7.5.0", + "entities": "^4.3.0", + "pino": "^8.6.1", + "ulid": "^2.3.0" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/util-utf8-browser": { - "version": "3.259.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", - "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", + "node_modules/@basemaps/shared/node_modules/@chunkd/fs": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@chunkd/fs/-/fs-11.2.0.tgz", + "integrity": "sha512-fwAkytCw8QnDgwZb/Sb6HlZ4n9EeKL3lV6KwkqFAxO2KtPDIyNcbyJ3H/6RjfWSHeyTA3+lHXEgT+XfyHpQfng==", "dependencies": { - "tslib": "^2.3.1" + "@chunkd/source": "^11.1.0", + "@chunkd/source-file": "^11.0.1", + "@chunkd/source-http": "^11.1.0", + "@chunkd/source-memory": "^11.0.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@aws-sdk/xml-builder": { - "version": "3.310.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.310.0.tgz", - "integrity": "sha512-TqELu4mOuSIKQCqj63fGVs86Yh+vBx5nHRpWKNUNhB2nPTpfbziTs5c1X358be3peVWA4wPxW7Nt53KIg1tnNw==", + "node_modules/@basemaps/shared/node_modules/@chunkd/source-file": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/@chunkd/source-file/-/source-file-11.0.1.tgz", + "integrity": "sha512-9DoA1djozuDpUklg0CRj/GBEJzvZeqwr1EIOTcEr3igDAf+UZ3y3lOgUaIWuNeATbwQ7Pdml2S6SLLAdAe45wQ==", "dependencies": { - "tslib": "^2.5.0" + "@chunkd/source": "^11.1.0" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@basemaps/config": { - "version": "7.7.0", - "resolved": "https://registry.npmjs.org/@basemaps/config/-/config-7.7.0.tgz", - "integrity": "sha512-LB2mQzkUnhJU4psFCSU2Xqu4dalpaXFOPRqPPZtE3OI04uOalMU7FYZOQeP9ubU1L4Gvh2AYwb03A8Ob0QtsTQ==", + "node_modules/@basemaps/shared/node_modules/@chunkd/source-http": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@chunkd/source-http/-/source-http-11.1.0.tgz", + "integrity": "sha512-vKCR/MwFuj7fAglym3BiSYromIFR65K9cIY3Z459B7HQR7aLATxQiQcxTfXNwFy+1/Xooe/6UXHrGs4EKcg6Vw==", "dependencies": { - "@basemaps/geo": "^7.5.0", - "base-x": "^4.0.0", - "zod": "^3.17.3" + "@chunkd/source": "^11.1.0" }, "engines": { - "node": ">=16.0.0" + "node": ">=18.0.0" } }, - "node_modules/@basemaps/geo": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@basemaps/geo/-/geo-7.5.0.tgz", - "integrity": "sha512-oHRs1lno7tPGnGvYTONsbstygVwShUv5K1VYCMe9OuNLGzF+EVQxDKL115Q4Xrkppv4H4Tiq7NglA95vpFAgTw==", + "node_modules/@basemaps/shared/node_modules/@chunkd/source-memory": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@chunkd/source-memory/-/source-memory-11.0.2.tgz", + "integrity": "sha512-K2yJTvLv+Z49YKGz43Gb2Q1/6ARtnUUh9EgVgqKZS/Ce9ND9IXkgmfyrYShY2/d9iJ4B7Nq1LZIJubDBhoYtww==", "dependencies": { - "@linzjs/tile-matrix-set": "^0.0.1", - "proj4": "^2.8.0" + "@chunkd/source": "^11.1.0" }, "engines": { "node": ">=16.0.0" @@ -977,6 +4566,99 @@ "@chunkd/source-google-cloud": "^10.0.4" } }, + "node_modules/@chunkd/fs-aws": { + "version": "11.3.0", + "resolved": "https://registry.npmjs.org/@chunkd/fs-aws/-/fs-aws-11.3.0.tgz", + "integrity": "sha512-Oq3knOYibBvBTGpjNrfol0x9w/H0Gyf0WbasX5NL6q8fJBrynR1vbBUnlT7s/DfSgu9AZLuMcD1dQa7t2ouNlA==", + "dependencies": { + "@aws-sdk/client-s3": "*", + "@aws-sdk/credential-providers": "*", + "@aws-sdk/lib-storage": "*", + "@chunkd/fs": "11.2.0", + "@chunkd/source-aws": "11.0.4" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@chunkd/fs-aws/node_modules/@chunkd/fs": { + "version": "11.2.0", + "resolved": "https://registry.npmjs.org/@chunkd/fs/-/fs-11.2.0.tgz", + "integrity": "sha512-fwAkytCw8QnDgwZb/Sb6HlZ4n9EeKL3lV6KwkqFAxO2KtPDIyNcbyJ3H/6RjfWSHeyTA3+lHXEgT+XfyHpQfng==", + "dependencies": { + "@chunkd/source": "^11.1.0", + "@chunkd/source-file": "^11.0.1", + "@chunkd/source-http": "^11.1.0", + "@chunkd/source-memory": "^11.0.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@chunkd/fs-aws/node_modules/@chunkd/source-aws": { + "version": "11.0.4", + "resolved": "https://registry.npmjs.org/@chunkd/source-aws/-/source-aws-11.0.4.tgz", + "integrity": "sha512-HvQ44z6Zz0f/tRFMyvfF4sQDCNXlorp5d6skei1mC0vZyMTn8BJfMkZKgvpDDUpyt5lGCxFREd3/LI140E0rZg==", + "dependencies": { + "@aws-sdk/client-s3": "*", + "@chunkd/source": "^11.1.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@chunkd/fs-aws/node_modules/@chunkd/source-file": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/@chunkd/source-file/-/source-file-11.0.1.tgz", + "integrity": "sha512-9DoA1djozuDpUklg0CRj/GBEJzvZeqwr1EIOTcEr3igDAf+UZ3y3lOgUaIWuNeATbwQ7Pdml2S6SLLAdAe45wQ==", + "dependencies": { + "@chunkd/source": "^11.1.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@chunkd/fs-aws/node_modules/@chunkd/source-http": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@chunkd/source-http/-/source-http-11.1.0.tgz", + "integrity": "sha512-vKCR/MwFuj7fAglym3BiSYromIFR65K9cIY3Z459B7HQR7aLATxQiQcxTfXNwFy+1/Xooe/6UXHrGs4EKcg6Vw==", + "dependencies": { + "@chunkd/source": "^11.1.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@chunkd/fs-aws/node_modules/@chunkd/source-memory": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/@chunkd/source-memory/-/source-memory-11.0.2.tgz", + "integrity": "sha512-K2yJTvLv+Z49YKGz43Gb2Q1/6ARtnUUh9EgVgqKZS/Ce9ND9IXkgmfyrYShY2/d9iJ4B7Nq1LZIJubDBhoYtww==", + "dependencies": { + "@chunkd/source": "^11.1.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@chunkd/middleware": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@chunkd/middleware/-/middleware-11.1.0.tgz", + "integrity": "sha512-xLEoDWDG3djf/plcsEsm0q6eAD3fFC/nJ6QV9234a+DE4iN8xuaqdHqtk9CRC877YfCpWZ1RDw3T62OUeqyWqQ==", + "dependencies": { + "@chunkd/source": "^11.1.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@chunkd/source": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/@chunkd/source/-/source-11.1.0.tgz", + "integrity": "sha512-3BcrHK4XDm3t4vDx1OisgcOZ05+EarEqwaGVIL7IMHUmkXwN/Aprlnr7aVP3BYcltgcCcfMd1pXQicbSrP563g==", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@chunkd/source-aws": { "version": "10.3.0", "resolved": "https://registry.npmjs.org/@chunkd/source-aws/-/source-aws-10.3.0.tgz", @@ -1062,6 +4744,18 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/@cotar/core": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@cotar/core/-/core-6.0.1.tgz", + "integrity": "sha512-919Q/76ypcLdsRWBsZEl3Lwsg2Wm1Zlr8XqbEaTps47QY8zMDsTwL3J/aVvb8DVgbBVONveEDOxj/04NsKrEqg==", + "dependencies": { + "@chunkd/source": "^11.0.0", + "@sindresorhus/fnv1a": "^3.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -1324,6 +5018,14 @@ "node": ">=16.0.0" } }, + "node_modules/@linzjs/metrics": { + "version": "7.5.0", + "resolved": "https://registry.npmjs.org/@linzjs/metrics/-/metrics-7.5.0.tgz", + "integrity": "sha512-HIvg9ynsRtFT+D2fJJ/rmcSaBEh/W4QQT68ckrvwjEsBxjBYCyiPXy8oU7t9KtuwXJbANKTNR4u+ZNBL2dOLHQ==", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@linzjs/style": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/@linzjs/style/-/style-5.4.0.tgz", @@ -1535,53 +5237,325 @@ "url": "https://opencollective.com/unts" } }, - "node_modules/@repeaterjs/repeater": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@repeaterjs/repeater/-/repeater-3.0.4.tgz", - "integrity": "sha512-AW8PKd6iX3vAZ0vA43nOUOnbq/X5ihgU+mSXXqunMkeQADGiqw/PY0JNeYtD5sr0PAy51YPgAPbDoeapv9r8WA==" + "node_modules/@repeaterjs/repeater": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@repeaterjs/repeater/-/repeater-3.0.4.tgz", + "integrity": "sha512-AW8PKd6iX3vAZ0vA43nOUOnbq/X5ihgU+mSXXqunMkeQADGiqw/PY0JNeYtD5sr0PAy51YPgAPbDoeapv9r8WA==" + }, + "node_modules/@sindresorhus/fnv1a": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@sindresorhus/fnv1a/-/fnv1a-3.1.0.tgz", + "integrity": "sha512-KV321z5m/0nuAg83W1dPLy85HpHDk7Sdi4fJbwvacWsEhAh+rZUW4ZfGcXmUIvjZg4ss2bcwNlRhJ7GBEUG08w==", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@smithy/abort-controller": { + "version": "2.0.12", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.12.tgz", + "integrity": "sha512-YIJyefe1mi3GxKdZxEBEuzYOeQ9xpYfqnFmWzojCssRAuR7ycxwpoRQgp965vuW426xUAQhCV5rCaWElQ7XsaA==", + "dependencies": { + "@smithy/types": "^2.4.0", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-4.0.0.tgz", + "integrity": "sha512-jSqRnZvkT4egkq/7b6/QRCNXmmYVcHwnJldqJ3IhVpQE2atObVJ137xmGeuGFhjFUr8gCEVAOKwSY79OvpbDaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/chunked-blob-reader-native": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.1.tgz", + "integrity": "sha512-VEYtPvh5rs/xlyqpm5NRnfYLZn+q0SRPELbvBV+C/G7IQ+ouTuo+NKKa3ShG5OaFR8NYVMXls9hPYLTvIKKDrQ==", + "dependencies": { + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/config-resolver": { + "version": "2.0.16", + "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-2.0.16.tgz", + "integrity": "sha512-1k+FWHQDt2pfpXhJsOmNMmlAZ3NUQ98X5tYsjQhVGq+0X6cOBMhfh6Igd0IX3Ut6lEO6DQAdPMI/blNr3JZfMQ==", + "dependencies": { + "@smithy/node-config-provider": "^2.1.3", + "@smithy/types": "^2.4.0", + "@smithy/util-config-provider": "^2.0.0", + "@smithy/util-middleware": "^2.0.5", + "tslib": "^2.5.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/core": { + "version": "2.5.4", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.5.4.tgz", + "integrity": "sha512-iFh2Ymn2sCziBRLPuOOxRPkuCx/2gBdXtBGuCUFLUe6bWYjKnhHyIPqGeNkLZ5Aco/5GjebRTBFiWID3sDbrKw==", + "dependencies": { + "@smithy/middleware-serde": "^3.0.10", + "@smithy/protocol-http": "^4.1.7", + "@smithy/types": "^3.7.1", + "@smithy/util-body-length-browser": "^3.0.0", + "@smithy/util-middleware": "^3.0.10", + "@smithy/util-stream": "^3.3.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/fetch-http-handler": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-4.1.1.tgz", + "integrity": "sha512-bH7QW0+JdX0bPBadXt8GwMof/jz0H28I84hU1Uet9ISpzUqXqRQ3fEZJ+ANPOhzSEczYvANNl3uDQDYArSFDtA==", + "dependencies": { + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/middleware-serde": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.10.tgz", + "integrity": "sha512-MnAuhh+dD14F428ubSJuRnmRsfOpxSzvRhaGVTvd/lrUDE3kxzCCmH8lnVTvoNQnV2BbJ4c15QwZ3UdQBtFNZA==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/node-http-handler": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.3.1.tgz", + "integrity": "sha512-fr+UAOMGWh6bn4YSEezBCpJn9Ukp9oR4D32sCjCo7U81evE11YePOQ58ogzyfgmjIO79YeOdfXXqr0jyhPQeMg==", + "dependencies": { + "@smithy/abort-controller": "^3.1.8", + "@smithy/protocol-http": "^4.1.7", + "@smithy/querystring-builder": "^3.0.10", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/protocol-http": { + "version": "4.1.7", + "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.1.7.tgz", + "integrity": "sha512-FP2LepWD0eJeOTm0SjssPcgqAlDFzOmRXqXmGhfIM52G7Lrox/pcpQf6RP4F21k0+O12zaqQt5fCDOeBtqY6Cg==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/querystring-builder": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.10.tgz", + "integrity": "sha512-nT9CQF3EIJtIUepXQuBFb8dxJi3WVZS3XfuDksxSCSn+/CzZowRLdhDn+2acbBv8R6eaJqPupoI/aRFIImNVPQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "@smithy/util-uri-escape": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/util-base64": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", + "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/util-body-length-browser": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", + "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } }, - "node_modules/@smithy/abort-controller": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.12.tgz", - "integrity": "sha512-YIJyefe1mi3GxKdZxEBEuzYOeQ9xpYfqnFmWzojCssRAuR7ycxwpoRQgp965vuW426xUAQhCV5rCaWElQ7XsaA==", + "node_modules/@smithy/core/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", "dependencies": { - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, - "node_modules/@smithy/chunked-blob-reader": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-2.0.0.tgz", - "integrity": "sha512-k+J4GHJsMSAIQPChGBrjEmGS+WbPonCXesoqP9fynIqjn7rdOThdH8FAeCmokP9mxTYKQAKoHCLPzNlm6gh7Wg==", + "node_modules/@smithy/core/node_modules/@smithy/util-middleware": { + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.10.tgz", + "integrity": "sha512-eJO+/+RsrG2RpmY68jZdwQtnfsxjmPxzMlQpnHKjFPwrYqvlcT+fHdT+ZVwcjlWSrByOhGr9Ff2GG17efc192A==", "dependencies": { - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@smithy/chunked-blob-reader-native": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-2.0.0.tgz", - "integrity": "sha512-HM8V2Rp1y8+1343tkZUKZllFhEQPNmpNdgFAncbTsxkZ18/gqjk23XXv3qGyXWp412f3o43ZZ1UZHVcHrpRnCQ==", + "node_modules/@smithy/core/node_modules/@smithy/util-stream": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.3.1.tgz", + "integrity": "sha512-Ff68R5lJh2zj+AUTvbAU/4yx+6QPRzg7+pI7M1FbtQHcRIp7xvguxVsQBKyB3fwiOwhAKu0lnNyYBaQfSW6TNw==", + "dependencies": { + "@smithy/fetch-http-handler": "^4.1.1", + "@smithy/node-http-handler": "^3.3.1", + "@smithy/types": "^3.7.1", + "@smithy/util-base64": "^3.0.0", + "@smithy/util-buffer-from": "^3.0.0", + "@smithy/util-hex-encoding": "^3.0.0", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/core/node_modules/@smithy/util-uri-escape": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", + "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", "dependencies": { - "@smithy/util-base64": "^2.0.0", - "tslib": "^2.5.0" + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, - "node_modules/@smithy/config-resolver": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-2.0.16.tgz", - "integrity": "sha512-1k+FWHQDt2pfpXhJsOmNMmlAZ3NUQ98X5tYsjQhVGq+0X6cOBMhfh6Igd0IX3Ut6lEO6DQAdPMI/blNr3JZfMQ==", + "node_modules/@smithy/core/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", "dependencies": { - "@smithy/node-config-provider": "^2.1.3", - "@smithy/types": "^2.4.0", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.5", - "tslib": "^2.5.0" + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" } }, "node_modules/@smithy/credential-provider-imds": { @@ -1611,54 +5585,143 @@ } }, "node_modules/@smithy/eventstream-serde-browser": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-2.0.12.tgz", - "integrity": "sha512-0pi8QlU/pwutNshoeJcbKR1p7Ie5STd8UFAMX5xhSoSJjNlxIv/OsHbF023jscMRN2Prrqd6ToGgdCnsZVQjvg==", + "version": "3.0.13", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.13.tgz", + "integrity": "sha512-Nee9m+97o9Qj6/XeLz2g2vANS2SZgAxV4rDBMKGHvFJHU/xz88x2RwCkwsvEwYjSX4BV1NG1JXmxEaDUzZTAtw==", "dependencies": { - "@smithy/eventstream-serde-universal": "^2.0.12", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/eventstream-serde-universal": "^3.0.12", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-browser/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-2.0.12.tgz", - "integrity": "sha512-I0XfwQkIX3gAnbrU5rLMkBSjTM9DHttdbLwf12CXmj7SSI5dT87PxtKLRrZGanaCMbdf2yCep+MW5/4M7IbvQA==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.10.tgz", + "integrity": "sha512-K1M0x7P7qbBUKB0UWIL5KOcyi6zqV5mPJoL0/o01HPJr0CSq3A9FYuJC6e11EX6hR8QTIR++DBiGrYveOu6trw==", "dependencies": { - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-config-resolver/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/eventstream-serde-node": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-2.0.12.tgz", - "integrity": "sha512-vf1vMHGOkG3uqN9x1zKOhnvW/XgvhJXWqjV6zZiT2FMjlEayugQ1mzpSqr7uf89+BzjTzuZKERmOsEAmewLbxw==", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.12.tgz", + "integrity": "sha512-kiZymxXvZ4tnuYsPSMUHe+MMfc4FTeFWJIc0Q5wygJoUQM4rVHNghvd48y7ppuulNMbuYt95ah71pYc2+o4JOA==", "dependencies": { - "@smithy/eventstream-serde-universal": "^2.0.12", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/eventstream-serde-universal": "^3.0.12", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-node/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/eventstream-serde-universal": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-2.0.12.tgz", - "integrity": "sha512-xZ3ZNpCxIND+q+UCy7y1n1/5VQEYicgSTNCcPqsKawX+Vd+6OcFX7gUHMyPzL8cZr+GdmJuxNleqHlH4giK2tw==", + "version": "3.0.12", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.12.tgz", + "integrity": "sha512-1i8ifhLJrOZ+pEifTlF0EfZzMLUGQggYQ6WmZ4d5g77zEKf7oZ0kvh1yKWHPjofvOwqrkwRDVuxuYC8wVd662A==", "dependencies": { - "@smithy/eventstream-codec": "^2.0.12", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/eventstream-codec": "^3.1.9", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal/node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal/node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/eventstream-codec": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.1.9.tgz", + "integrity": "sha512-F574nX0hhlNOjBnP+noLtsPFqXnWh2L0+nZKCwcu7P7J8k+k+rdIDs+RMnrMwrzhUE4mwMgyN0cYnEn0G8yrnQ==", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^3.7.1", + "@smithy/util-hex-encoding": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/util-hex-encoding": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", + "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/fetch-http-handler": { @@ -1674,14 +5737,25 @@ } }, "node_modules/@smithy/hash-blob-browser": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-2.0.12.tgz", - "integrity": "sha512-riLnV16f27yyePX8UF0deRHAeccUK8SrOxyTykSTrnVkgS3DsjNapZtTbd8OGNKEbI60Ncdb5GwN3rHZudXvog==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.1.9.tgz", + "integrity": "sha512-wOu78omaUuW5DE+PVWXiRKWRZLecARyP3xcq5SmkXUw9+utgN8HnSnBfrjL2B/4ZxgqPjaAJQkC/+JHf1ITVaQ==", "dependencies": { - "@smithy/chunked-blob-reader": "^2.0.0", - "@smithy/chunked-blob-reader-native": "^2.0.0", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/chunked-blob-reader": "^4.0.0", + "@smithy/chunked-blob-reader-native": "^3.0.1", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/hash-blob-browser/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/hash-node": { @@ -1699,16 +5773,62 @@ } }, "node_modules/@smithy/hash-stream-node": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-2.0.12.tgz", - "integrity": "sha512-x/DrSynPKrW0k00q7aZ/vy531a3mRw79mOajHp+cIF0TrA1SqEMFoy/B8X0XtoAtlJWt/vvgeDNqt/KAeaAqMw==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.1.9.tgz", + "integrity": "sha512-3XfHBjSP3oDWxLmlxnt+F+FqXpL3WlXs+XXaB6bV9Wo8BBu87fK1dSEsyH7Z4ZHRmwZ4g9lFMdf08m9hoX1iRA==", "dependencies": { - "@smithy/types": "^2.4.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/hash-stream-node/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/hash-stream-node/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/hash-stream-node/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/hash-stream-node/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/invalid-dependency": { @@ -1732,13 +5852,59 @@ } }, "node_modules/@smithy/md5-js": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-2.0.12.tgz", - "integrity": "sha512-OgDt+Xnrw+W5z3MSl5KZZzebqmXrYl9UdbCiBYnnjErmNywwSjV6QB/Oic3/7hnsPniSU81n7Rvlhz2kH4EREQ==", + "version": "3.0.10", + "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.10.tgz", + "integrity": "sha512-m3bv6dApflt3fS2Y1PyWPUtRP7iuBlvikEOGwu0HsCZ0vE7zcIX+dBoh3e+31/rddagw8nj92j0kJg2TfV+SJA==", "dependencies": { - "@smithy/types": "^2.4.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" + "@smithy/types": "^3.7.1", + "@smithy/util-utf8": "^3.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@smithy/md5-js/node_modules/@smithy/is-array-buffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", + "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/md5-js/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/md5-js/node_modules/@smithy/util-buffer-from": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", + "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", + "dependencies": { + "@smithy/is-array-buffer": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/md5-js/node_modules/@smithy/util-utf8": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", + "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", + "dependencies": { + "@smithy/util-buffer-from": "^3.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@smithy/middleware-content-length": { @@ -2144,16 +6310,39 @@ } }, "node_modules/@smithy/util-waiter": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-2.0.12.tgz", - "integrity": "sha512-3sENmyVa1NnOPoiT2NCApPmu7ukP7S/v7kL9IxNmnygkDldn7/yK0TP42oPJLwB2k3mospNsSePIlqdXEUyPHA==", + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.1.9.tgz", + "integrity": "sha512-/aMXPANhMOlMPjfPtSrDfPeVP8l56SJlz93xeiLmhLe5xvlXA5T3abZ2ilEsDEPeY9T/wnN/vNGn9wa1SbufWA==", "dependencies": { - "@smithy/abort-controller": "^2.0.12", - "@smithy/types": "^2.4.0", - "tslib": "^2.5.0" + "@smithy/abort-controller": "^3.1.8", + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" }, "engines": { - "node": ">=14.0.0" + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-waiter/node_modules/@smithy/abort-controller": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.1.8.tgz", + "integrity": "sha512-+3DOBcUn5/rVjlxGvUPKc416SExarAQ+Qe0bqk30YSUjbepwpS7QN0cyKUSifvLJhdMZ0WPzPP5ymut0oonrpQ==", + "dependencies": { + "@smithy/types": "^3.7.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@smithy/util-waiter/node_modules/@smithy/types": { + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.7.1.tgz", + "integrity": "sha512-XKLcLXZY7sUQgvvWyeaL/qwNPp6V3dWcUjqrQKjSb+tzYiCy340R/c64LV5j+Tnb2GhmunEX0eou+L+m2hJNYA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" } }, "node_modules/@tootallnate/once": { @@ -2296,6 +6485,11 @@ "prettier": "*" } }, + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "7.14.1", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.14.1.tgz", @@ -3192,6 +7386,17 @@ "optional": true, "peer": true }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-abstract": { "version": "1.23.3", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.23.3.tgz", @@ -5035,6 +9240,14 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/mnemonist": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "dependencies": { + "obliterator": "^1.6.1" + } + }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -5191,6 +9404,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/obliterator": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==" + }, "node_modules/ol": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/ol/-/ol-8.1.0.tgz", diff --git a/package.json b/package.json index 2c26e00f..ad898865 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,10 @@ "@aws-sdk/client-s3": "^3.440.0", "@aws-sdk/credential-providers": "^3.438.0", "@aws-sdk/lib-storage": "^3.440.0", + "@basemaps/cogify": "^7.12.0", "@basemaps/config": "^7.7.0", "@basemaps/geo": "^7.5.0", + "@basemaps/shared": "^7.12.0", "@chunkd/fs": "^10.0.9", "@chunkd/source-aws-v3": "^10.1.3", "@cogeotiff/core": "^9.0.3", diff --git a/src/commands/basemaps-topo-import/__test__/list-jobs.test.ts b/src/commands/basemaps-topo-import/__test__/list-jobs.test.ts deleted file mode 100644 index 753a9950..00000000 --- a/src/commands/basemaps-topo-import/__test__/list-jobs.test.ts +++ /dev/null @@ -1,30 +0,0 @@ -import assert from 'node:assert'; -import { describe, it } from 'node:test'; - -import { extractMapSheetName } from '../list-jobs.js'; - -describe('extractMapSheetName', () => { - const FakeDomain = 's3://topographic/fake-domain'; - const FakeFiles = [ - { actual: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: 'MB07_v1-00' }, - { actual: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: 'MB07_v1-00' }, - { actual: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: 'MB07_v1-00' }, - { actual: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: 'MB07_v1-00' }, - { actual: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`, expected: 'AX32ptsAX31AY31AY32_v1-00' }, - { actual: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`, expected: 'AZ36ptsAZ35BA35BA36_v1-00' }, - ]; - - it('Should parse the correct MapSheet Names', async () => { - for (const file of FakeFiles) { - const output = extractMapSheetName(file.actual); - assert.equal(output, file.expected); - } - }); - - it('Should not able to parse a version from file', async () => { - const wrongFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`]; - for (const file of wrongFiles) { - assert.throws(() => extractMapSheetName(file), new Error('Version not found in the file name')); - } - }); -}); diff --git a/src/commands/basemaps-topo-import/list-jobs.ts b/src/commands/basemaps-topo-import/list-jobs.ts deleted file mode 100644 index 64dfd741..00000000 --- a/src/commands/basemaps-topo-import/list-jobs.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { fsa } from '@chunkd/fs'; -import { command, option, string } from 'cmd-ts'; -import path from 'path'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; -import { isTiff } from '../tileindex-validate/tileindex.validate.js'; - -interface Output { - /** - * Input Topo Raster tiff files for processing - */ - input: string[]; - - /** - * Output name for the processed tiff files - */ - output: string; -} - -/** - * List all the tiffs in a directory and output the list of names to process. - * Outputs a json file with the list of files to process. - * - * @example - * [ - * { - * "output": "CJ10_v1-00", - * "input": [ - * "s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" - * ] - * } - * ] - * - * @param location: Location of the source files - * @example s3://topographic-upload/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ - */ -export const basemapsListTopoJobs = command({ - name: 'list-topo-jobs', - description: 'List input files and validate there are no duplicates.', - version: CliInfo.version, - args: { - config, - verbose, - forceOutput, - location: option({ - type: string, - long: 'location', - description: 'Location of the source files', - }), - }, - async handler(args) { - const startTime = performance.now(); - registerCli(this, args); - logger.info('ListJobs:Start'); - - const files = await fsa.toArray(fsa.list(args.location)); - const outputs: Output[] = []; - for (const file of files) { - logger.info({ file }, 'ListJobs:File'); - if (!isTiff(file)) continue; - - const output = extractMapSheetName(file); - outputs.push({ output, input: [file] }); - } - - if (outputs.length === 0) throw new Error('No tiff files found in the location'); - - if (args.forceOutput || isArgo()) { - await fsa.write('/tmp/list-topo-jobs/file-list.json', JSON.stringify(outputs, null, 2)); - } - - logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); - }, -}); - -export function extractMapSheetName(file: string): string { - const url = tryParseUrl(file); - const filePath = path.parse(url.href); - const tileName = filePath.name; - - // pull map sheet name off front of tileName (e.g. CJ10) - const mapSheetName = tileName.split('_')[0]; - // pull version off end of tileName (e.g. v1-0) - const version = tileName.match(/v(\d)-(\d\d)/)?.[0]; - if (version == null) throw new Error('Version not found in the file name'); - - const output = `${mapSheetName}_${version}`; - logger.info({ mapSheetName, version, output }, 'ListJobs:Output'); - return output; -} diff --git a/src/commands/basemaps-topo-map-import/__test__/import-topographic-maps.test.ts b/src/commands/basemaps-topo-map-import/__test__/import-topographic-maps.test.ts new file mode 100644 index 00000000..40a00dfd --- /dev/null +++ b/src/commands/basemaps-topo-map-import/__test__/import-topographic-maps.test.ts @@ -0,0 +1,37 @@ +import assert from 'node:assert'; +import { describe, it } from 'node:test'; + +import { extractMapSheetNameWithVersion } from '../import-topographic-maps.js'; + +describe('extractMapSheetName', () => { + const FakeDomain = 's3://topographic/fake-domain'; + const FakeFiles = [ + { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { input: `${FakeDomain}/MB07_GRIDLESS_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { input: `${FakeDomain}/MB07_TIFFv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { input: `${FakeDomain}/MB07_TIFF_600v1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, + { + input: `${FakeDomain}/AX32ptsAX31AY31AY32_GeoTifv1-00.tif`, + expected: { mapCode: 'AX32ptsAX31AY31AY32', version: 'v1-00' }, + }, + { + input: `${FakeDomain}/AZ36ptsAZ35BA35BA36_GeoTifv1-00.tif`, + expected: { mapCode: 'AZ36ptsAZ35BA35BA36', version: 'v1-00' }, + }, + ]; + + it('Should parse the correct MapSheet Names', async () => { + for (const file of FakeFiles) { + const output = extractMapSheetNameWithVersion(file.input); + assert.equal(output.mapCode, file.expected.mapCode, 'Map code does not match'); + assert.equal(output.version, file.expected.version, 'Version does not match'); + } + }); + + it('Should not able to parse a version from file', async () => { + const wrongFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`]; + for (const file of wrongFiles) { + assert.throws(() => extractMapSheetNameWithVersion(file), new Error('Version not found in the file name')); + } + }); +}); diff --git a/src/commands/basemaps-topo-map-import/gdal-commands.ts b/src/commands/basemaps-topo-map-import/gdal-commands.ts new file mode 100644 index 00000000..fac1275c --- /dev/null +++ b/src/commands/basemaps-topo-map-import/gdal-commands.ts @@ -0,0 +1,32 @@ +import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; + +import { urlToString } from '../common.js'; + +export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { + return { + command: 'gdal_translate', + output, + args: [ + ['-q'], // Supress non-error output + ['-of', 'COG'], // Output format + ['-stats'], // Force stats (re)computation + ['-a_srs', `EPSG:2193`], // Projection override + // creation options (-co) + ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS + ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes + ['-co', 'BIGTIFF=NO'], + ['-co', 'OVERVIEWS=IGNORE_EXISTING'], + ['-co', `BLOCKSIZE=512`], + ['-co', `COMPRESS=webp`], + ['-co', `QUALITY=100`], + ['-co', `OVERVIEW_COMPRESS=webp`], + ['-co', `OVERVIEW_RESAMPLING=lanczos`], + ['-co', `OVERVIEW_QUALITY=90`], + urlToString(input), + urlToString(output), + ] + .filter((f) => f != null) + .flat() + .map(String), + }; +} diff --git a/src/commands/basemaps-topo-map-import/import-topographic-maps.ts b/src/commands/basemaps-topo-map-import/import-topographic-maps.ts new file mode 100644 index 00000000..9c841eb3 --- /dev/null +++ b/src/commands/basemaps-topo-map-import/import-topographic-maps.ts @@ -0,0 +1,135 @@ +import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { fsa } from '@chunkd/fs'; +import { command, option, string } from 'cmd-ts'; +import path from 'path'; + +import { CliInfo } from '../../cli.info.js'; +import { logger } from '../../log.js'; +import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; +import { isTiff } from '../tileindex-validate/tileindex.validate.js'; +import { gdalBuildCogCommands } from './gdal-commands.js'; + +interface ImportJob { + /** + * Input location of the topographic raster tiff file. + * + * @example "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + */ + input: string; + + /** + * Map code of the topographic raster tiff file. + * + * @example "CJ10" + */ + mapCode: string; + + /** + * Version of the topographic raster tiff file. + * + * @example "v1-00" + */ + version: string; + + /** + * Output location for the processed tiff file. + * + * @example TODO + */ + output: string; +} + +/** + * List all the tiffs in a directory for topographic maps and create cogs for each. + * + * @param source: Location of the source files + * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ + * + * @param target: Location of the target path + */ +export const importTopographicMaps = command({ + name: 'import-topographic-maps', + description: 'List input topographic files and run dgal to standardize and import into target.', + version: CliInfo.version, + args: { + config, + verbose, + forceOutput, + source: option({ + type: string, + long: 'source', + description: 'Location of the source files', + }), + target: option({ + type: string, + long: 'target', + description: 'Target location for the output files', + }), + }, + async handler(args) { + const startTime = performance.now(); + registerCli(this, args); + logger.info('ListJobs:Start'); + + // extract all file paths from the source directory + const files = await fsa.toArray(fsa.list(args.source)); + + // map each file path into an ImportJob + const jobs: ImportJob[] = []; + + for (const file of files) { + logger.info({ file }, 'ListJobs:File'); + if (!isTiff(file)) continue; + + const { mapCode, version } = extractMapSheetNameWithVersion(file); + const job: ImportJob = { + input: file, + mapCode, + version, + output: fsa.join(args.target, `${mapCode}_${version}.tiff`), + }; + + jobs.push(job); + } + + if (jobs.length === 0) throw new Error('No tiff files found in the location'); + + // run gdal_translate for each job + for (const job of jobs) { + const command = gdalBuildCogCommands(tryParseUrl(job.input), tryParseUrl(job.output)); + logger.level = 'debug'; + await new GdalRunner(command).run(logger); + } + + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); + }, +}); + +/** + * Extract the map sheet name and version from the provided path. + * Throws an error if either detail cannot be parsed. + * + * @param file: Location of the source file + * + * @example + * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + * returns: { mapCode: "CJ10", version: "v1-00" } + * + * @returns + */ +export function extractMapSheetNameWithVersion(file: string): { mapCode: string; version: string } { + const url = tryParseUrl(file); + const filePath = path.parse(url.href); + const tileName = filePath.name; + + // extract map code from head of the filepath (e.g. CJ10) + const mapCode = tileName.split('_')[0]; + if (mapCode == null) throw new Error('Map sheet not found in the file name'); + + // extract version from tail of the filepath (e.g. v1-0) + const version = tileName.match(/v(\d)-(\d\d)/)?.[0]; + if (version == null) throw new Error('Version not found in the file name'); + + logger.info({ mapCode, version }, 'ListJobs:Output'); + return { mapCode, version }; +} diff --git a/src/commands/index.ts b/src/commands/index.ts index 9773af1d..09911eb1 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -3,7 +3,7 @@ import { subcommands } from 'cmd-ts'; import { CliInfo } from '../cli.info.js'; import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; -import { basemapsListTopoJobs } from './basemaps-topo-import/list-jobs.js'; +import { importTopographicMaps } from './basemaps-topo-map-import/import-topographic-maps.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; import { commandGeneratePath } from './generate-path/path.generate.js'; @@ -49,7 +49,7 @@ export const AllCommands = { cmds: { 'create-pr': basemapsCreatePullRequest, 'create-mapsheet': basemapsCreateMapSheet, - 'list-topo-jobs': basemapsListTopoJobs, + 'import-topographic-maps': importTopographicMaps, }, }), 'pretty-print': commandPrettyPrint, From b5499e3f650b576a05167229c576088918e5628b Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 26 Nov 2024 14:14:49 +1300 Subject: [PATCH 06/47] Build on a dgal container --- Dockerfile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1ce55934..0f17c4d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,14 @@ -FROM node:20-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 +FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.0 RUN apt-get update && apt-get install openssh-client git -y +RUN apt-get install -y ca-certificates curl gnupg +RUN mkdir -p /etc/apt/keyrings +RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +ENV NODE_MAJOR=20 +RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list + +RUN apt-get update +RUN apt-get install -y nodejs WORKDIR /app From 5f21781629a79a05260527e5f03c2dd91d30bddb Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 27 Nov 2024 11:31:12 +1300 Subject: [PATCH 07/47] WIP --- package-lock.json | 14 +++++ package.json | 1 + .../import-topographic-maps.ts | 54 ++++++++++++++++++- 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index af911148..035726ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@aws-sdk/lib-storage": "^3.440.0", "@basemaps/cogify": "^7.12.0", "@basemaps/config": "^7.7.0", + "@basemaps/config-loader": "^7.12.0", "@basemaps/geo": "^7.5.0", "@basemaps/shared": "^7.12.0", "@chunkd/fs": "^10.0.9", @@ -4457,6 +4458,19 @@ "node": ">=16.0.0" } }, + "node_modules/@basemaps/config-loader": { + "version": "7.12.0", + "resolved": "https://registry.npmjs.org/@basemaps/config-loader/-/config-loader-7.12.0.tgz", + "integrity": "sha512-Biiw+Py8IDX74HTXx3HsLvz0jGmVPVYY1n2+Dd919ZYUZn9Q+Q46blaj8Xq+GXKSKy3F8E8sq6KGA734FB4P+A==", + "dependencies": { + "@basemaps/config": "^7.12.0", + "@basemaps/geo": "^7.12.0", + "@basemaps/shared": "^7.12.0" + }, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@basemaps/geo": { "version": "7.12.0", "resolved": "https://registry.npmjs.org/@basemaps/geo/-/geo-7.12.0.tgz", diff --git a/package.json b/package.json index ad898865..652aa288 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "@aws-sdk/lib-storage": "^3.440.0", "@basemaps/cogify": "^7.12.0", "@basemaps/config": "^7.7.0", + "@basemaps/config-loader": "^7.12.0", "@basemaps/geo": "^7.5.0", "@basemaps/shared": "^7.12.0", "@chunkd/fs": "^10.0.9", diff --git a/src/commands/basemaps-topo-map-import/import-topographic-maps.ts b/src/commands/basemaps-topo-map-import/import-topographic-maps.ts index 9c841eb3..38480bfd 100644 --- a/src/commands/basemaps-topo-map-import/import-topographic-maps.ts +++ b/src/commands/basemaps-topo-map-import/import-topographic-maps.ts @@ -1,6 +1,14 @@ +import { tmpdir } from 'node:os'; + import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { createTileCover, TileCoverContext } from '@basemaps/cogify/build/tile.cover.js'; +import { ConfigProviderMemory } from '@basemaps/config'; +import { initConfigFromUrls } from '@basemaps/config-loader'; +import { Nztm2000QuadTms } from '@basemaps/geo'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; import { fsa } from '@chunkd/fs'; import { command, option, string } from 'cmd-ts'; +import pLimit from 'p-limit'; import path from 'path'; import { CliInfo } from '../../cli.info.js'; @@ -9,6 +17,8 @@ import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../commo import { isTiff } from '../tileindex-validate/tileindex.validate.js'; import { gdalBuildCogCommands } from './gdal-commands.js'; +const Q = pLimit(10); + interface ImportJob { /** * Input location of the topographic raster tiff file. @@ -37,6 +47,8 @@ interface ImportJob { * @example TODO */ output: string; + + latest: boolean; } /** @@ -74,9 +86,28 @@ export const importTopographicMaps = command({ // extract all file paths from the source directory const files = await fsa.toArray(fsa.list(args.source)); + const mem = new ConfigProviderMemory(); + const cfg = await initConfigFromUrls( + mem, + files.map((f) => tryParseUrl(f)), + ); + if (cfg.imagery.length === 0) throw new Error('No imagery found'); + const im = cfg.imagery[0]; + if (im == null) throw new Error('No imagery found'); + logger.info({ files: im.files.length, title: im.title }, 'Imagery:Loaded'); + + const ctx: TileCoverContext = { + id: CliId, + imagery: im, + tileMatrix: Nztm2000QuadTms, + logger, + preset: 'webp', + }; + + const res = await createTileCover(ctx); + // map each file path into an ImportJob const jobs: ImportJob[] = []; - for (const file of files) { logger.info({ file }, 'ListJobs:File'); if (!isTiff(file)) continue; @@ -87,15 +118,36 @@ export const importTopographicMaps = command({ mapCode, version, output: fsa.join(args.target, `${mapCode}_${version}.tiff`), + latest: false, }; + // Tag latest version for the job + + jobs.push(job); } + // Upload the stac files into target location + if (jobs.length === 0) throw new Error('No tiff files found in the location'); + // Prepare the temporary folder for the source files and outputs source:tmp/source/*tif tmp/outputs:tmp/output/*tiff + const tmpPath = path.join(tmpdir(), CliId); + const tmpURL = tryParseUrl(tmpPath); + const tmpFolder = tmpURL.href.endsWith('/') ? new URL(tmpURL.href) : new URL(`${tmpURL.href}/`); + + // async download and dgal commands plimit of 10 + Q(async () => { + // download the source files + // run dgal_traslate for source and output + // fsa.write output to target location + }); + // run gdal_translate for each job for (const job of jobs) { + // Download the source file + for (const src of files) sources.register(new URL(src.href, i.url), i.item.id); + const command = gdalBuildCogCommands(tryParseUrl(job.input), tryParseUrl(job.output)); logger.level = 'debug'; await new GdalRunner(command).run(logger); From 9dcf6308e7690a5ce8d7c43e6d576917c72b6be5 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 27 Nov 2024 16:19:23 +1300 Subject: [PATCH 08/47] Add stac creation and download for gdal --- .../__test__/import-topographic-maps.test.ts | 0 .../gdal-commands.ts | 0 .../import-topographic-maps.ts | 262 ++++++++++++++++++ .../import-topographic-maps.ts | 187 ------------- src/commands/index.ts | 2 +- 5 files changed, 263 insertions(+), 188 deletions(-) rename src/commands/{basemaps-topo-map-import => basemaps-topo-import}/__test__/import-topographic-maps.test.ts (100%) rename src/commands/{basemaps-topo-map-import => basemaps-topo-import}/gdal-commands.ts (100%) create mode 100644 src/commands/basemaps-topo-import/import-topographic-maps.ts delete mode 100644 src/commands/basemaps-topo-map-import/import-topographic-maps.ts diff --git a/src/commands/basemaps-topo-map-import/__test__/import-topographic-maps.test.ts b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts similarity index 100% rename from src/commands/basemaps-topo-map-import/__test__/import-topographic-maps.test.ts rename to src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts diff --git a/src/commands/basemaps-topo-map-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts similarity index 100% rename from src/commands/basemaps-topo-map-import/gdal-commands.ts rename to src/commands/basemaps-topo-import/gdal-commands.ts diff --git a/src/commands/basemaps-topo-import/import-topographic-maps.ts b/src/commands/basemaps-topo-import/import-topographic-maps.ts new file mode 100644 index 00000000..5dca8527 --- /dev/null +++ b/src/commands/basemaps-topo-import/import-topographic-maps.ts @@ -0,0 +1,262 @@ +import { tmpdir } from 'node:os'; + +import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; +import { Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { fsa } from '@chunkd/fs'; +import { Tiff } from '@cogeotiff/core'; +import { command, option, string } from 'cmd-ts'; +import { mkdir, rm } from 'fs/promises'; +import pLimit from 'p-limit'; +import path from 'path'; +import { StacCollection, StacItem } from 'stac-ts'; +import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; + +import { CliInfo } from '../../cli.info.js'; +import { logger } from '../../log.js'; +import { isArgo } from '../../utils/argo.js'; +import { findBoundingBox } from '../../utils/geotiff.js'; +import { HashTransform } from '../../utils/hash.stream.js'; +import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; +import { gdalBuildCogCommands } from './gdal-commands.js'; + +const Q = pLimit(10); + +/** + * List all the tiffs in a directory for topographic maps and create cogs for each. + * + * @param source: Location of the source files + * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ + * + * @param target: Location of the target path + */ +export const importTopographicMaps = command({ + name: 'import-topographic-maps', + description: 'List input topographic files and run dgal to standardize and import into target.', + version: CliInfo.version, + args: { + config, + verbose, + forceOutput, + title: option({ + type: string, + long: 'title', + description: 'Imported imagery title', + }), + source: option({ + type: string, + long: 'source', + description: 'Location of the source files', + }), + target: option({ + type: string, + long: 'target', + description: 'Target location for the output files', + }), + }, + async handler(args) { + const startTime = performance.now(); + registerCli(this, args); + logger.info('ListJobs:Start'); + + const items = await loadTiffsToCreateStacs(args.source, args.target, args.title, args.forceOutput); + if (items.length === 0) throw new Error('No Stac items created'); + + const tmpPath = path.join(tmpdir(), CliId); + const tmpURL = tryParseUrl(tmpPath); + const tmpFolder = tmpURL.href.endsWith('/') ? new URL(tmpURL.href) : new URL(`${tmpURL.href}/`); + await Promise.all( + items.map((item) => + Q(async () => { + await createCogs(item, args.target, tmpFolder); + }), + ), + ); + + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); + }, +}); + +/** + * Extract the map code and version from the provided path. + * Throws an error if either detail cannot be parsed. + * + * @param file: filepath from which to extract the map code and version + * + * @example + * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + * returns: { mapCode: "CJ10", version: "v1-00" } + * + * @returns an object containing the map code and version + */ +export function extractMapSheetNameWithVersion(file: string): { mapCode: string; version: string } { + const url = tryParseUrl(file); + const filePath = path.parse(url.href); + const fileName = filePath.name; + + // extract map code from head of the file name (e.g. CJ10) + const mapCode = fileName.split('_')[0]; + if (mapCode == null) throw new Error('Map sheet not found in the file name'); + + // extract version from tail of the file name (e.g. v1-0) + const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; + if (version == null) throw new Error('Version not found in the file name'); + + logger.info({ mapCode, version }, 'ListJobs:Output'); + return { mapCode, version }; +} + +/** + * @param source: Source directory URL from which to load tiff files + * @example TODO + * + * @param target: Destination directory URL into which to save the STAC collection and item JSON files + * @example TODO + * + * @param title: The title of the collection + * @example "New Zealand Topo50 Map Series (Gridless)" + * + * @returns an array of StacItem objects + */ +async function loadTiffsToCreateStacs( + source: string, + target: string, + title: string, + force: boolean, +): Promise { + // extract all file paths from the source directory and convert them into URL objects + logger.info({ source }, 'LoadTiffs:Start'); + const files = await fsa.toArray(fsa.list(source)); + const fileUrls = files.map((f) => tryParseUrl(f)); + + const tiffs = await loadTiffsFromPaths(fileUrls, Q); + const projection = Projection.get(Nztm2000QuadTms); + const cliDate = new Date().toISOString(); + + const items: StacItem[] = []; + let imageryBound: Bounds | undefined; + logger.info({ tiffs: tiffs.length }, 'CreateStac:Start'); + const brokenTiffs = new Map(); + for (const tiff of tiffs) { + const source = tiff.source.url.href; + const { mapCode, version } = extractMapSheetNameWithVersion(source); + const tileName = `${mapCode}_${version}`; + let bounds; + try { + bounds = Bounds.fromBbox(await findBoundingBox(tiff)); + } catch (e) { + brokenTiffs.set(tileName, tiff); + continue; + } + if (imageryBound == null) { + imageryBound = bounds; + } else { + imageryBound = imageryBound.union(bounds); + } + + logger.info({ tileName }, 'CreateStac:Item'); + const item: StacItem = { + id: tileName, + type: 'Feature', + collection: CliId, + stac_version: '1.0.0', + stac_extensions: [], + geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, + bbox: projection.boundsToWgs84BoundingBox(bounds), + links: [ + { href: `./${tileName}.json`, rel: 'self' }, + { href: './collection.json', rel: 'collection' }, + { href: './collection.json', rel: 'parent' }, + { href: source, rel: 'linz_basemaps:source', type: 'image/tiff; application=geotiff' }, + ], + properties: { + version, + datetime: cliDate, + start_datetime: undefined, + end_datetime: undefined, + 'proj:epsg': projection.epsg.code, + created_at: cliDate, + updated_at: cliDate, + 'linz:lifecycle': 'ongoing', + 'linz:geospatial_category': 'topographic-maps', + 'linz:region': 'new-zealand', + 'linz:security_classification': 'unclassified', + 'linz:slug': 'topo50', + }, + assets: {}, + }; + items.push(item); + } + if (imageryBound == null) throw new Error('No imagery bounds found'); + + logger.info({ items: items.length }, 'CreateStac:Collection'); + const collection: StacCollection = { + id: CliId, + type: 'Collection', + stac_version: '1.0.0', + stac_extensions: [], + license: 'CC-BY-4.0', + title, + description: 'Topographic maps of New Zealand', + providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], + extent: { + spatial: { bbox: [projection.boundsToWgs84BoundingBox(imageryBound)] }, + // Default the temporal time today if no times were found as it is required for STAC + temporal: { interval: [[cliDate, null]] }, + }, + links: items.map((item) => { + return { href: `./${item.id}.json`, rel: 'item', type: 'application/json' }; + }), + }; + + if (force || isArgo()) { + logger.info({ target }, 'CreateStac:Output'); + logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); + for (const item of items) { + await fsa.write(fsa.join(target, `${item.id}.json`), JSON.stringify(item, null, 2)); + } + await fsa.write(fsa.join(target, 'collection.json'), JSON.stringify(collection, null, 2)); + } + + await fsa.write( + fsa.join(`${target}/broken/`, 'broken.json'), + JSON.stringify(Array.from(brokenTiffs.keys()), null, 2), + ); + + return items; +} + +async function createCogs(item: StacItem, target: string, tmpFolder: URL): Promise { + try { + // Extract the source URL from the item + logger.info({ item: item.id }, 'CogCreation:Start'); + const source = item.links.find((l) => l.rel === 'linz_basemaps:source')?.href; + if (source == null) throw new Error('No source file found in the item'); + await mkdir(tmpFolder, { recursive: true }); + + // Download the source file + logger.info({ item: item.id }, 'CogCreation:Download'); + const url = tryParseUrl(source); + const filePath = path.parse(url.href); + const fileName = filePath.base; + const hashStreamSource = fsa.stream(source).pipe(new HashTransform('sha256')); + const input = fsa.join(tmpFolder.href, fileName); + await fsa.write(input, hashStreamSource); + + // run gdal_translate for each job + logger.info({ item: item.id }, 'CogCreation:gdal_translate'); + const output = fsa.join(tmpFolder.href, `${item.id}.tiff`); + const command = gdalBuildCogCommands(tryParseUrl(input), tryParseUrl(output)); + await new GdalRunner(command).run(logger); + + // fsa.write output to target location + logger.info({ item: item.id }, 'CogCreation:Output'); + const readStream = fsa.stream(output).pipe(new HashTransform('sha256')); + await fsa.write(fsa.join(target, `${item.id}.tiff`), readStream); + } finally { + // Cleanup the temporary folder once everything is done + logger.info({ path: tmpFolder }, 'CogCreation:Cleanup'); + await rm(tmpFolder, { recursive: true, force: true }); + } +} diff --git a/src/commands/basemaps-topo-map-import/import-topographic-maps.ts b/src/commands/basemaps-topo-map-import/import-topographic-maps.ts deleted file mode 100644 index 38480bfd..00000000 --- a/src/commands/basemaps-topo-map-import/import-topographic-maps.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { tmpdir } from 'node:os'; - -import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { createTileCover, TileCoverContext } from '@basemaps/cogify/build/tile.cover.js'; -import { ConfigProviderMemory } from '@basemaps/config'; -import { initConfigFromUrls } from '@basemaps/config-loader'; -import { Nztm2000QuadTms } from '@basemaps/geo'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; -import { fsa } from '@chunkd/fs'; -import { command, option, string } from 'cmd-ts'; -import pLimit from 'p-limit'; -import path from 'path'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; -import { isTiff } from '../tileindex-validate/tileindex.validate.js'; -import { gdalBuildCogCommands } from './gdal-commands.js'; - -const Q = pLimit(10); - -interface ImportJob { - /** - * Input location of the topographic raster tiff file. - * - * @example "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" - */ - input: string; - - /** - * Map code of the topographic raster tiff file. - * - * @example "CJ10" - */ - mapCode: string; - - /** - * Version of the topographic raster tiff file. - * - * @example "v1-00" - */ - version: string; - - /** - * Output location for the processed tiff file. - * - * @example TODO - */ - output: string; - - latest: boolean; -} - -/** - * List all the tiffs in a directory for topographic maps and create cogs for each. - * - * @param source: Location of the source files - * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ - * - * @param target: Location of the target path - */ -export const importTopographicMaps = command({ - name: 'import-topographic-maps', - description: 'List input topographic files and run dgal to standardize and import into target.', - version: CliInfo.version, - args: { - config, - verbose, - forceOutput, - source: option({ - type: string, - long: 'source', - description: 'Location of the source files', - }), - target: option({ - type: string, - long: 'target', - description: 'Target location for the output files', - }), - }, - async handler(args) { - const startTime = performance.now(); - registerCli(this, args); - logger.info('ListJobs:Start'); - - // extract all file paths from the source directory - const files = await fsa.toArray(fsa.list(args.source)); - - const mem = new ConfigProviderMemory(); - const cfg = await initConfigFromUrls( - mem, - files.map((f) => tryParseUrl(f)), - ); - if (cfg.imagery.length === 0) throw new Error('No imagery found'); - const im = cfg.imagery[0]; - if (im == null) throw new Error('No imagery found'); - logger.info({ files: im.files.length, title: im.title }, 'Imagery:Loaded'); - - const ctx: TileCoverContext = { - id: CliId, - imagery: im, - tileMatrix: Nztm2000QuadTms, - logger, - preset: 'webp', - }; - - const res = await createTileCover(ctx); - - // map each file path into an ImportJob - const jobs: ImportJob[] = []; - for (const file of files) { - logger.info({ file }, 'ListJobs:File'); - if (!isTiff(file)) continue; - - const { mapCode, version } = extractMapSheetNameWithVersion(file); - const job: ImportJob = { - input: file, - mapCode, - version, - output: fsa.join(args.target, `${mapCode}_${version}.tiff`), - latest: false, - }; - - // Tag latest version for the job - - - jobs.push(job); - } - - // Upload the stac files into target location - - if (jobs.length === 0) throw new Error('No tiff files found in the location'); - - // Prepare the temporary folder for the source files and outputs source:tmp/source/*tif tmp/outputs:tmp/output/*tiff - const tmpPath = path.join(tmpdir(), CliId); - const tmpURL = tryParseUrl(tmpPath); - const tmpFolder = tmpURL.href.endsWith('/') ? new URL(tmpURL.href) : new URL(`${tmpURL.href}/`); - - // async download and dgal commands plimit of 10 - Q(async () => { - // download the source files - // run dgal_traslate for source and output - // fsa.write output to target location - }); - - // run gdal_translate for each job - for (const job of jobs) { - // Download the source file - for (const src of files) sources.register(new URL(src.href, i.url), i.item.id); - - const command = gdalBuildCogCommands(tryParseUrl(job.input), tryParseUrl(job.output)); - logger.level = 'debug'; - await new GdalRunner(command).run(logger); - } - - logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); - }, -}); - -/** - * Extract the map sheet name and version from the provided path. - * Throws an error if either detail cannot be parsed. - * - * @param file: Location of the source file - * - * @example - * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" - * returns: { mapCode: "CJ10", version: "v1-00" } - * - * @returns - */ -export function extractMapSheetNameWithVersion(file: string): { mapCode: string; version: string } { - const url = tryParseUrl(file); - const filePath = path.parse(url.href); - const tileName = filePath.name; - - // extract map code from head of the filepath (e.g. CJ10) - const mapCode = tileName.split('_')[0]; - if (mapCode == null) throw new Error('Map sheet not found in the file name'); - - // extract version from tail of the filepath (e.g. v1-0) - const version = tileName.match(/v(\d)-(\d\d)/)?.[0]; - if (version == null) throw new Error('Version not found in the file name'); - - logger.info({ mapCode, version }, 'ListJobs:Output'); - return { mapCode, version }; -} diff --git a/src/commands/index.ts b/src/commands/index.ts index 09911eb1..790939fa 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -3,7 +3,7 @@ import { subcommands } from 'cmd-ts'; import { CliInfo } from '../cli.info.js'; import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; -import { importTopographicMaps } from './basemaps-topo-map-import/import-topographic-maps.js'; +import { importTopographicMaps } from './basemaps-topo-import/import-topographic-maps.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; import { commandGeneratePath } from './generate-path/path.generate.js'; From 8768af950b8d0992658a2d12cf1ae8766f6b4db9 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 28 Nov 2024 10:34:51 +1300 Subject: [PATCH 09/47] Update to use basemaps/shared fsa --- .../import-topographic-maps.ts | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/commands/basemaps-topo-import/import-topographic-maps.ts b/src/commands/basemaps-topo-import/import-topographic-maps.ts index 5dca8527..dc29f299 100644 --- a/src/commands/basemaps-topo-import/import-topographic-maps.ts +++ b/src/commands/basemaps-topo-import/import-topographic-maps.ts @@ -3,8 +3,8 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; import { Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { fsa } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; -import { fsa } from '@chunkd/fs'; import { Tiff } from '@cogeotiff/core'; import { command, option, string } from 'cmd-ts'; import { mkdir, rm } from 'fs/promises'; @@ -127,10 +127,9 @@ async function loadTiffsToCreateStacs( ): Promise { // extract all file paths from the source directory and convert them into URL objects logger.info({ source }, 'LoadTiffs:Start'); - const files = await fsa.toArray(fsa.list(source)); - const fileUrls = files.map((f) => tryParseUrl(f)); - - const tiffs = await loadTiffsFromPaths(fileUrls, Q); + const sourceUrl = tryParseUrl(source); + const files = await fsa.toArray(fsa.list(sourceUrl)); + const tiffs = await loadTiffsFromPaths(files, Q); const projection = Projection.get(Nztm2000QuadTms); const cliDate = new Date().toISOString(); @@ -214,15 +213,15 @@ async function loadTiffsToCreateStacs( logger.info({ target }, 'CreateStac:Output'); logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); for (const item of items) { - await fsa.write(fsa.join(target, `${item.id}.json`), JSON.stringify(item, null, 2)); + const itemPath = new URL(`${item.id}.json`, target); + await fsa.write(itemPath, JSON.stringify(item, null, 2)); } - await fsa.write(fsa.join(target, 'collection.json'), JSON.stringify(collection, null, 2)); + const collectionPath = new URL('collection.json', target); + await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); } - await fsa.write( - fsa.join(`${target}/broken/`, 'broken.json'), - JSON.stringify(Array.from(brokenTiffs.keys()), null, 2), - ); + const brokenPath = new URL('broken.json', `${target}/broken/`); + await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); return items; } @@ -237,23 +236,24 @@ async function createCogs(item: StacItem, target: string, tmpFolder: URL): Promi // Download the source file logger.info({ item: item.id }, 'CogCreation:Download'); - const url = tryParseUrl(source); - const filePath = path.parse(url.href); + const sourceUrl = tryParseUrl(source); + const filePath = path.parse(sourceUrl.href); const fileName = filePath.base; - const hashStreamSource = fsa.stream(source).pipe(new HashTransform('sha256')); - const input = fsa.join(tmpFolder.href, fileName); - await fsa.write(input, hashStreamSource); + const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); + const inputPath = new URL(fileName, tmpFolder); + await fsa.write(inputPath, hashStreamSource); // run gdal_translate for each job logger.info({ item: item.id }, 'CogCreation:gdal_translate'); - const output = fsa.join(tmpFolder.href, `${item.id}.tiff`); - const command = gdalBuildCogCommands(tryParseUrl(input), tryParseUrl(output)); + const tempPath = new URL(`${item.id}.tiff`, tmpFolder); + const command = gdalBuildCogCommands(inputPath, tempPath); await new GdalRunner(command).run(logger); // fsa.write output to target location logger.info({ item: item.id }, 'CogCreation:Output'); - const readStream = fsa.stream(output).pipe(new HashTransform('sha256')); - await fsa.write(fsa.join(target, `${item.id}.tiff`), readStream); + const readStream = fsa.readStream(tempPath).pipe(new HashTransform('sha256')); + const outputPath = new URL(`${item.id}.tiff`, target); + await fsa.write(outputPath, readStream); } finally { // Cleanup the temporary folder once everything is done logger.info({ path: tmpFolder }, 'CogCreation:Cleanup'); From 83db0cb0e528740be0917ac47d6879bda28bbcc5 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 28 Nov 2024 10:48:01 +1300 Subject: [PATCH 10/47] update the tmp folder for each cog processing --- .../import-topographic-maps.ts | 31 ++++++++++--------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/commands/basemaps-topo-import/import-topographic-maps.ts b/src/commands/basemaps-topo-import/import-topographic-maps.ts index dc29f299..b450217e 100644 --- a/src/commands/basemaps-topo-import/import-topographic-maps.ts +++ b/src/commands/basemaps-topo-import/import-topographic-maps.ts @@ -60,7 +60,12 @@ export const importTopographicMaps = command({ registerCli(this, args); logger.info('ListJobs:Start'); - const items = await loadTiffsToCreateStacs(args.source, args.target, args.title, args.forceOutput); + const items = await loadTiffsToCreateStacs( + tryParseUrl(args.source), + tryParseUrl(args.target), + args.title, + args.forceOutput, + ); if (items.length === 0) throw new Error('No Stac items created'); const tmpPath = path.join(tmpdir(), CliId); @@ -69,7 +74,7 @@ export const importTopographicMaps = command({ await Promise.all( items.map((item) => Q(async () => { - await createCogs(item, args.target, tmpFolder); + await createCogs(item, tryParseUrl(args.target), tmpFolder); }), ), ); @@ -119,16 +124,10 @@ export function extractMapSheetNameWithVersion(file: string): { mapCode: string; * * @returns an array of StacItem objects */ -async function loadTiffsToCreateStacs( - source: string, - target: string, - title: string, - force: boolean, -): Promise { +async function loadTiffsToCreateStacs(source: URL, target: URL, title: string, force: boolean): Promise { // extract all file paths from the source directory and convert them into URL objects logger.info({ source }, 'LoadTiffs:Start'); - const sourceUrl = tryParseUrl(source); - const files = await fsa.toArray(fsa.list(sourceUrl)); + const files = await fsa.toArray(fsa.list(source)); const tiffs = await loadTiffsFromPaths(files, Q); const projection = Projection.get(Nztm2000QuadTms); const cliDate = new Date().toISOString(); @@ -220,13 +219,14 @@ async function loadTiffsToCreateStacs( await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); } - const brokenPath = new URL('broken.json', `${target}/broken/`); + const brokenPath = new URL('./broken/broken.json', target); await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); return items; } -async function createCogs(item: StacItem, target: string, tmpFolder: URL): Promise { +async function createCogs(item: StacItem, target: URL, tmp: URL): Promise { + const tmpFolder = new URL(item.id, tmp); try { // Extract the source URL from the item logger.info({ item: item.id }, 'CogCreation:Start'); @@ -235,12 +235,13 @@ async function createCogs(item: StacItem, target: string, tmpFolder: URL): Promi await mkdir(tmpFolder, { recursive: true }); // Download the source file - logger.info({ item: item.id }, 'CogCreation:Download'); + const sourceUrl = tryParseUrl(source); const filePath = path.parse(sourceUrl.href); const fileName = filePath.base; const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); const inputPath = new URL(fileName, tmpFolder); + logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); await fsa.write(inputPath, hashStreamSource); // run gdal_translate for each job @@ -256,7 +257,7 @@ async function createCogs(item: StacItem, target: string, tmpFolder: URL): Promi await fsa.write(outputPath, readStream); } finally { // Cleanup the temporary folder once everything is done - logger.info({ path: tmpFolder }, 'CogCreation:Cleanup'); - await rm(tmpFolder, { recursive: true, force: true }); + logger.info({ path: tmpFolder.href }, 'CogCreation:Cleanup'); + await rm(tmpFolder.href, { recursive: true, force: true }); } } From 11d3fc5b9f983bf737b2cc6b354fba767bd6be70 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 28 Nov 2024 15:48:07 +1300 Subject: [PATCH 11/47] Group the task to process cogs in seperated nodes --- .../__test__/import-topographic-maps.test.ts | 2 +- .../import-topographic-maps.ts | 263 ------------- .../basemaps-topo-import/topo-cog-creation.ts | 115 ++++++ .../topo-stac-creation.ts | 365 ++++++++++++++++++ src/commands/group/group.ts | 2 +- src/commands/index.ts | 6 +- 6 files changed, 486 insertions(+), 267 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/import-topographic-maps.ts create mode 100644 src/commands/basemaps-topo-import/topo-cog-creation.ts create mode 100644 src/commands/basemaps-topo-import/topo-stac-creation.ts diff --git a/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts index 40a00dfd..3710dc44 100644 --- a/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts +++ b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { extractMapSheetNameWithVersion } from '../import-topographic-maps.js'; +import { extractMapSheetNameWithVersion } from '../topo-stac-creation.js'; describe('extractMapSheetName', () => { const FakeDomain = 's3://topographic/fake-domain'; diff --git a/src/commands/basemaps-topo-import/import-topographic-maps.ts b/src/commands/basemaps-topo-import/import-topographic-maps.ts deleted file mode 100644 index b450217e..00000000 --- a/src/commands/basemaps-topo-import/import-topographic-maps.ts +++ /dev/null @@ -1,263 +0,0 @@ -import { tmpdir } from 'node:os'; - -import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; -import { Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; -import { fsa } from '@basemaps/shared'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; -import { Tiff } from '@cogeotiff/core'; -import { command, option, string } from 'cmd-ts'; -import { mkdir, rm } from 'fs/promises'; -import pLimit from 'p-limit'; -import path from 'path'; -import { StacCollection, StacItem } from 'stac-ts'; -import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { findBoundingBox } from '../../utils/geotiff.js'; -import { HashTransform } from '../../utils/hash.stream.js'; -import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; -import { gdalBuildCogCommands } from './gdal-commands.js'; - -const Q = pLimit(10); - -/** - * List all the tiffs in a directory for topographic maps and create cogs for each. - * - * @param source: Location of the source files - * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ - * - * @param target: Location of the target path - */ -export const importTopographicMaps = command({ - name: 'import-topographic-maps', - description: 'List input topographic files and run dgal to standardize and import into target.', - version: CliInfo.version, - args: { - config, - verbose, - forceOutput, - title: option({ - type: string, - long: 'title', - description: 'Imported imagery title', - }), - source: option({ - type: string, - long: 'source', - description: 'Location of the source files', - }), - target: option({ - type: string, - long: 'target', - description: 'Target location for the output files', - }), - }, - async handler(args) { - const startTime = performance.now(); - registerCli(this, args); - logger.info('ListJobs:Start'); - - const items = await loadTiffsToCreateStacs( - tryParseUrl(args.source), - tryParseUrl(args.target), - args.title, - args.forceOutput, - ); - if (items.length === 0) throw new Error('No Stac items created'); - - const tmpPath = path.join(tmpdir(), CliId); - const tmpURL = tryParseUrl(tmpPath); - const tmpFolder = tmpURL.href.endsWith('/') ? new URL(tmpURL.href) : new URL(`${tmpURL.href}/`); - await Promise.all( - items.map((item) => - Q(async () => { - await createCogs(item, tryParseUrl(args.target), tmpFolder); - }), - ), - ); - - logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); - }, -}); - -/** - * Extract the map code and version from the provided path. - * Throws an error if either detail cannot be parsed. - * - * @param file: filepath from which to extract the map code and version - * - * @example - * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" - * returns: { mapCode: "CJ10", version: "v1-00" } - * - * @returns an object containing the map code and version - */ -export function extractMapSheetNameWithVersion(file: string): { mapCode: string; version: string } { - const url = tryParseUrl(file); - const filePath = path.parse(url.href); - const fileName = filePath.name; - - // extract map code from head of the file name (e.g. CJ10) - const mapCode = fileName.split('_')[0]; - if (mapCode == null) throw new Error('Map sheet not found in the file name'); - - // extract version from tail of the file name (e.g. v1-0) - const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; - if (version == null) throw new Error('Version not found in the file name'); - - logger.info({ mapCode, version }, 'ListJobs:Output'); - return { mapCode, version }; -} - -/** - * @param source: Source directory URL from which to load tiff files - * @example TODO - * - * @param target: Destination directory URL into which to save the STAC collection and item JSON files - * @example TODO - * - * @param title: The title of the collection - * @example "New Zealand Topo50 Map Series (Gridless)" - * - * @returns an array of StacItem objects - */ -async function loadTiffsToCreateStacs(source: URL, target: URL, title: string, force: boolean): Promise { - // extract all file paths from the source directory and convert them into URL objects - logger.info({ source }, 'LoadTiffs:Start'); - const files = await fsa.toArray(fsa.list(source)); - const tiffs = await loadTiffsFromPaths(files, Q); - const projection = Projection.get(Nztm2000QuadTms); - const cliDate = new Date().toISOString(); - - const items: StacItem[] = []; - let imageryBound: Bounds | undefined; - logger.info({ tiffs: tiffs.length }, 'CreateStac:Start'); - const brokenTiffs = new Map(); - for (const tiff of tiffs) { - const source = tiff.source.url.href; - const { mapCode, version } = extractMapSheetNameWithVersion(source); - const tileName = `${mapCode}_${version}`; - let bounds; - try { - bounds = Bounds.fromBbox(await findBoundingBox(tiff)); - } catch (e) { - brokenTiffs.set(tileName, tiff); - continue; - } - if (imageryBound == null) { - imageryBound = bounds; - } else { - imageryBound = imageryBound.union(bounds); - } - - logger.info({ tileName }, 'CreateStac:Item'); - const item: StacItem = { - id: tileName, - type: 'Feature', - collection: CliId, - stac_version: '1.0.0', - stac_extensions: [], - geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, - bbox: projection.boundsToWgs84BoundingBox(bounds), - links: [ - { href: `./${tileName}.json`, rel: 'self' }, - { href: './collection.json', rel: 'collection' }, - { href: './collection.json', rel: 'parent' }, - { href: source, rel: 'linz_basemaps:source', type: 'image/tiff; application=geotiff' }, - ], - properties: { - version, - datetime: cliDate, - start_datetime: undefined, - end_datetime: undefined, - 'proj:epsg': projection.epsg.code, - created_at: cliDate, - updated_at: cliDate, - 'linz:lifecycle': 'ongoing', - 'linz:geospatial_category': 'topographic-maps', - 'linz:region': 'new-zealand', - 'linz:security_classification': 'unclassified', - 'linz:slug': 'topo50', - }, - assets: {}, - }; - items.push(item); - } - if (imageryBound == null) throw new Error('No imagery bounds found'); - - logger.info({ items: items.length }, 'CreateStac:Collection'); - const collection: StacCollection = { - id: CliId, - type: 'Collection', - stac_version: '1.0.0', - stac_extensions: [], - license: 'CC-BY-4.0', - title, - description: 'Topographic maps of New Zealand', - providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], - extent: { - spatial: { bbox: [projection.boundsToWgs84BoundingBox(imageryBound)] }, - // Default the temporal time today if no times were found as it is required for STAC - temporal: { interval: [[cliDate, null]] }, - }, - links: items.map((item) => { - return { href: `./${item.id}.json`, rel: 'item', type: 'application/json' }; - }), - }; - - if (force || isArgo()) { - logger.info({ target }, 'CreateStac:Output'); - logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); - for (const item of items) { - const itemPath = new URL(`${item.id}.json`, target); - await fsa.write(itemPath, JSON.stringify(item, null, 2)); - } - const collectionPath = new URL('collection.json', target); - await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); - } - - const brokenPath = new URL('./broken/broken.json', target); - await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); - - return items; -} - -async function createCogs(item: StacItem, target: URL, tmp: URL): Promise { - const tmpFolder = new URL(item.id, tmp); - try { - // Extract the source URL from the item - logger.info({ item: item.id }, 'CogCreation:Start'); - const source = item.links.find((l) => l.rel === 'linz_basemaps:source')?.href; - if (source == null) throw new Error('No source file found in the item'); - await mkdir(tmpFolder, { recursive: true }); - - // Download the source file - - const sourceUrl = tryParseUrl(source); - const filePath = path.parse(sourceUrl.href); - const fileName = filePath.base; - const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); - const inputPath = new URL(fileName, tmpFolder); - logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); - await fsa.write(inputPath, hashStreamSource); - - // run gdal_translate for each job - logger.info({ item: item.id }, 'CogCreation:gdal_translate'); - const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const command = gdalBuildCogCommands(inputPath, tempPath); - await new GdalRunner(command).run(logger); - - // fsa.write output to target location - logger.info({ item: item.id }, 'CogCreation:Output'); - const readStream = fsa.readStream(tempPath).pipe(new HashTransform('sha256')); - const outputPath = new URL(`${item.id}.tiff`, target); - await fsa.write(outputPath, readStream); - } finally { - // Cleanup the temporary folder once everything is done - logger.info({ path: tmpFolder.href }, 'CogCreation:Cleanup'); - await rm(tmpFolder.href, { recursive: true, force: true }); - } -} diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts new file mode 100644 index 00000000..3ac2872e --- /dev/null +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -0,0 +1,115 @@ +import { tmpdir } from 'node:os'; + +import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { fsa } from '@basemaps/shared'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { command, option, optional, restPositionals, string } from 'cmd-ts'; +import { mkdir, rm } from 'fs/promises'; +import pLimit from 'p-limit'; +import path from 'path'; +import { StacItem } from 'stac-ts'; + +import { CliInfo } from '../../cli.info.js'; +import { logger } from '../../log.js'; +import { HashTransform } from '../../utils/hash.stream.js'; +import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; +import { loadInput } from '../group/group.js'; +import { gdalBuildCogCommands } from './gdal-commands.js'; +const Q = pLimit(10); + +/** + * List all the tiffs in a directory for topographic maps and create cogs for each. + * + * @param source: Location of the source files + * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ + * + * @param target: Location of the target path + */ +export const topoCogCreation = command({ + name: 'topo-cog-creation', + description: 'Get the list of topo cog stac items and creating cog for them.', + version: CliInfo.version, + args: { + config, + verbose, + forceOutput, + inputs: restPositionals({ + type: string, + displayName: 'items', + description: 'list of items to group, can be a JSON array', + }), + fromFile: option({ + type: optional(string), + long: 'from-file', + description: 'JSON file to load inputs from, must be a JSON Array', + }), + }, + async handler(args) { + const startTime = performance.now(); + registerCli(this, args); + logger.info('ListJobs:Start'); + // Load the items for processing + const inputs: string[] = []; + for (const input of args.inputs) inputs.push(...loadInput(input)); + if (args.fromFile && (await fsa.exists(tryParseUrl(args.fromFile)))) { + const input = await fsa.readJson(tryParseUrl(args.fromFile)); + if (Array.isArray(input)) inputs.push(...input); + } + + if (inputs.length === 0) { + logger.error('Group:Error:Empty'); + process.exit(1); + } + + // Prepare temporary folder for dgdal to create cog + const tmpPath = path.join(tmpdir(), CliId); + const tmpURL = tryParseUrl(tmpPath); + const tmpFolder = tmpURL.href.endsWith('/') ? new URL(tmpURL.href) : new URL(`${tmpURL.href}/`); + await Promise.all( + inputs.map((input) => + Q(async () => { + await createCogs(tryParseUrl(input), tmpFolder); + }), + ), + ); + + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); + }, +}); + +async function createCogs(input: URL, tmp: URL): Promise { + const item = await fsa.readJson(input); + const tmpFolder = new URL(item.id, tmp); + try { + // Extract the source URL from the item + logger.info({ item: item.id }, 'CogCreation:Start'); + const source = item.links.find((l) => l.rel === 'linz_basemaps:source')?.href; + if (source == null) throw new Error('No source file found in the item'); + await mkdir(tmpFolder, { recursive: true }); + + // Download the source file + const sourceUrl = tryParseUrl(source); + const filePath = path.parse(sourceUrl.href); + const fileName = filePath.base; + const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); + const inputPath = new URL(fileName, tmpFolder); + logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); + await fsa.write(inputPath, hashStreamSource); + + // run gdal_translate for each job + logger.info({ item: item.id }, 'CogCreation:gdal_translate'); + const tempPath = new URL(`${item.id}.tiff`, tmpFolder); + const command = gdalBuildCogCommands(inputPath, tempPath); + await new GdalRunner(command).run(logger); + + // fsa.write output to target location + logger.info({ item: item.id }, 'CogCreation:Output'); + const readStream = fsa.readStream(tempPath).pipe(new HashTransform('sha256')); + const outputPath = tryParseUrl(input.href.replace('.json', '.tiff')); + await fsa.write(outputPath, readStream); + } finally { + // Cleanup the temporary folder once everything is done + logger.info({ path: tmpFolder.href }, 'CogCreation:Cleanup'); + await rm(tmpFolder.href, { recursive: true, force: true }); + } +} diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts new file mode 100644 index 00000000..0a2513c2 --- /dev/null +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -0,0 +1,365 @@ +import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; +import { BoundingBox, Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { fsa } from '@basemaps/shared'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { Tiff } from '@cogeotiff/core'; +import { command, option, string } from 'cmd-ts'; +import pLimit from 'p-limit'; +import path from 'path'; +import { StacCollection, StacItem } from 'stac-ts'; +import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; + +import { CliInfo } from '../../cli.info.js'; +import { logger } from '../../log.js'; +import { isArgo } from '../../utils/argo.js'; +import { findBoundingBox } from '../../utils/geotiff.js'; +import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; + +const Q = pLimit(10); +const projection = Projection.get(Nztm2000QuadTms); +const cliDate = new Date().toISOString(); +const brokenTiffs = new Map(); + +/** + * List all the tiffs in a directory for topographic maps and create cogs for each. + * + * @param source: Location of the source files + * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ + * + * @param target: Location of the target path + */ +export const topoStacCreation = command({ + name: 'topo-stac-creation', + description: 'List input topographic files and run dgal to standardize and import into target.', + version: CliInfo.version, + args: { + config, + verbose, + forceOutput, + title: option({ + type: string, + long: 'title', + description: 'Imported imagery title', + }), + source: option({ + type: string, + long: 'source', + description: 'Location of the source files', + }), + target: option({ + type: string, + long: 'target', + description: 'Target location for the output files', + }), + scale: option({ + type: string, + long: 'scale', + description: 'topo50 or topo250', + }), + }, + async handler(args) { + const startTime = performance.now(); + registerCli(this, args); + logger.info('ListJobs:Start'); + + const sourceURL = tryParseUrl(args.source); + const targetURL = tryParseUrl(args.target); + + const { latest, others } = await loadTiffsToCreateStacs( + sourceURL, + targetURL, + args.title, + args.forceOutput, + args.scale, + ); + if (latest.length === 0 || others.length === 0) throw new Error('No Stac items created'); + + const paths: string[] = []; + others.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, targetURL).href)); + latest.forEach((item) => paths.push(new URL(`${args.scale}-latest/${item.id}.json`, targetURL).href)); + + // write stac items into an JSON array + await fsa.write(tryParseUrl(`${args.target}/tiles.json`), JSON.stringify(paths, null, 2)); + + logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); + }, +}); + +interface VersionedTiff { + version: string; + tiff: Tiff; +} + +/** + * @param source: Source directory URL from which to load tiff files + * @example TODO + * + * @param target: Destination directory URL into which to save the STAC collection and item JSON files + * @example TODO + * + * @param title: The title of the collection + * @example "New Zealand Topo50 Map Series (Gridless)" + * + * @returns an array of StacItem objects + */ +async function loadTiffsToCreateStacs( + source: URL, + target: URL, + title: string, + force: boolean, + scale: string, +): Promise<{ latest: StacItem[]; others: StacItem[] }> { + // extract all file paths from the source directory and convert them into URL objects + logger.info({ source }, 'LoadTiffs:Start'); + const files = (await fsa.toArray(fsa.list(source))).slice(0, 20); + const tiffs = await loadTiffsFromPaths(files, Q); + + // we need to assign each tiff to a group based on its map code (e.g. AB01) + // for each group, we then need to identify the latest version and set it aside from the rest + // the latest version will have special metadata, whereas the rest will have similar metadata + + // group the tiffs by map code + // + // { + // "AB01": ["v1-00", "v1-01", "v2-00"] + // "CD01": ["v1-00", "v2-00", "v2-01"] + // } + const versionsByMapCode: Map = new Map(); + + for (const tiff of tiffs) { + const source = tiff.source.url.href; + const { mapCode, version } = extractMapSheetNameWithVersion(source); + + const entry = versionsByMapCode.get(mapCode); + + if (entry == null) { + versionsByMapCode.set(mapCode, [{ version, tiff }]); + } else { + entry.push({ version, tiff }); + } + } + + // for each group, identify the latest version + // + // { + // "AB01": { latest: "v2-00", others: ["v1-00", "v1-01"] } + // "CD01": { latest: "v2-01", others: ["v1-00", "v2-00"] } + // } + const groupsByMapCode: Map = new Map(); + + for (const [mapCode, versions] of versionsByMapCode.entries()) { + const sorted = versions.sort((a, b) => a.version.localeCompare(b.version)); + + const latest = sorted[sorted.length - 1]; + if (latest == null) throw new Error(); + + const others = sorted.filter((version) => version !== latest); + + groupsByMapCode.set(mapCode, { latest, others }); + } + + const latestStacs: StacItem[] = []; + const otherStacs: StacItem[] = []; + let imageryBound: Bounds | undefined; + + logger.info({ tiffs: tiffs.length }, 'CreateStac:Start'); + + for (const [mapCode, { latest, others }] of groupsByMapCode.entries()) { + const stacItems = await createStacItems(mapCode, latest, others, target, scale); + latestStacs.push(stacItems.latest); + otherStacs.push(...stacItems.others); + + if (imageryBound == null) { + imageryBound = stacItems.bounds; + } else { + imageryBound = imageryBound.union(stacItems.bounds); + } + } + + if (imageryBound == null) throw new Error('No imagery bounds found'); + + // Create collection json for all topo50 items + const latestCollection = createStacCollection(title, imageryBound, latestStacs); + const collection = createStacCollection(title, imageryBound, otherStacs); + + await writeStacFiles(new URL(`${scale}-latest/`, target), force, latestStacs, latestCollection); + await writeStacFiles(new URL(`${scale}/`, target), force, otherStacs, collection); + + return { latest: latestStacs, others: otherStacs }; +} + +/** + * Extract the map code and version from the provided path. + * Throws an error if either detail cannot be parsed. + * + * @param file: filepath from which to extract the map code and version + * + * @example + * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + * returns: { mapCode: "CJ10", version: "v1-00" } + * + * @returns an object containing the map code and version + */ +export function extractMapSheetNameWithVersion(file: string): { mapCode: string; version: string } { + const url = tryParseUrl(file); + const filePath = path.parse(url.href); + const fileName = filePath.name; + + // extract map code from head of the file name (e.g. CJ10) + const mapCode = fileName.split('_')[0]; + if (mapCode == null) throw new Error('Map sheet not found in the file name'); + + // extract version from tail of the file name (e.g. v1-0) + const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; + if (version == null) throw new Error('Version not found in the file name'); + + logger.info({ mapCode, version }, 'ListJobs:Output'); + return { mapCode, version }; +} + +/** + * This function needs to create two groups: + * - StacItem objects that will live in the Topo50 directory + * - StacItem objects that will live in the Topo250 directory + * + * All versions need a StacItem object that lives in the Topo50 dir + * The latest version needs a second StacItem object that lives in the Topo250 dir + */ +async function createStacItems( + mapCode: string, + latest: VersionedTiff, + others: VersionedTiff[], + target: URL, + scale: string, +): Promise<{ latest: StacItem; others: StacItem[]; bounds: Bounds }> { + const latestStacItem = await createBaseStacItem(mapCode, mapCode, latest.version, latest.tiff); + const othersStacItems = await Promise.all( + [...others, latest].map(({ version, tiff }) => createBaseStacItem(`${mapCode}_${version}`, mapCode, version, tiff)), + ); + + // need to do the part where they add special fields to each group + const latestURL = new URL(`${scale}/${mapCode}_${latest.version}.json`, target); + + // add link to others pointing to latest + othersStacItems.forEach((item) => { + item?.links.push({ + href: latestURL.href, + rel: 'latest-version', + type: 'application/json', + }); + }); + + // add link to latest referencing its copy that will live in topo50 dir + latestStacItem?.links.push({ + href: latestURL.href, + rel: 'derived_from', + type: 'application/json', + }); + + // since all of the given tiffs are for the same map code, + // their bounds should be the same, so we can grab the bounds of the latest + try { + const bounds = Bounds.fromBbox(await findBoundingBox(latest.tiff)); + + if (latestStacItem == null) { + throw new Error(`Error creating StacItem for latest version of ${mapCode} sheets`); + } + + return { latest: latestStacItem, others: othersStacItems.flatMap((item) => (item ? item : [])), bounds }; + } catch (e) { + throw new Error(`Couldn't extract bounds from the ${mapCode}'s latest map sheet`); + } +} + +async function createBaseStacItem(id: string, mapCode: string, version: string, tiff: Tiff): Promise { + const source = tiff.source.url.href; + + let bounds; + try { + bounds = Bounds.fromBbox(await findBoundingBox(tiff)); + } catch (e) { + brokenTiffs.set(id, tiff); + return null; + } + + logger.info({ id }, 'CreateStac:Item'); + const item: StacItem = { + id: id, + type: 'Feature', + collection: CliId, + stac_version: '1.0.0', + stac_extensions: [], + geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, + bbox: projection.boundsToWgs84BoundingBox(bounds), + links: [ + { href: `./${id}.json`, rel: 'self' }, + { href: './collection.json', rel: 'collection' }, + { href: './collection.json', rel: 'parent' }, + { href: source, rel: 'linz_basemaps:source', type: 'image/tiff; application=geotiff' }, + ], + properties: { + mapCode, + version, + datetime: cliDate, + start_datetime: undefined, + end_datetime: undefined, + 'proj:epsg': projection.epsg.code, + created_at: cliDate, + updated_at: cliDate, + 'linz:lifecycle': 'ongoing', + 'linz:geospatial_category': 'topographic-maps', + 'linz:region': 'new-zealand', + 'linz:security_classification': 'unclassified', + 'linz:slug': 'topo50', + }, + assets: {}, + }; + + return item; +} + +function createStacCollection(title: string, imageryBound: BoundingBox, items: StacItem[]): StacCollection { + logger.info({ items: items.length }, 'CreateStac:Collection'); + const collection: StacCollection = { + id: CliId, + type: 'Collection', + stac_version: '1.0.0', + stac_extensions: [], + license: 'CC-BY-4.0', + title, + description: 'Topographic maps of New Zealand', + providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], + extent: { + spatial: { bbox: [projection.boundsToWgs84BoundingBox(imageryBound)] }, + // Default the temporal time today if no times were found as it is required for STAC + temporal: { interval: [[cliDate, null]] }, + }, + links: items.map((item) => { + return { href: `./${item.id}.json`, rel: 'item', type: 'application/json', latest: true }; + }), + }; + + return collection; +} + +async function writeStacFiles( + target: URL, + force: boolean, + items: StacItem[], + collection: StacCollection, +): Promise { + // Create collection json for all topo50-latest items. + if (force || isArgo()) { + logger.info({ target }, 'CreateStac:Output'); + logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); + for (const item of items) { + const itemPath = new URL(`${item.id}.json`, target); + await fsa.write(itemPath, JSON.stringify(item, null, 2)); + } + const collectionPath = new URL('collection.json', target); + await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); + } + + const brokenPath = new URL('/tmp/topo-stac-creation/output/broken.json', target); + await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); +} diff --git a/src/commands/group/group.ts b/src/commands/group/group.ts index b20c70c0..a786804c 100644 --- a/src/commands/group/group.ts +++ b/src/commands/group/group.ts @@ -23,7 +23,7 @@ export function groupItems(items: T[], groupSize: number): T[][] { } /** Normalize an input as either a JSON array or just an array */ -function loadInput(x: string): string[] { +export function loadInput(x: string): string[] { if (x.startsWith('[')) return JSON.parse(x) as string[]; return [x]; } diff --git a/src/commands/index.ts b/src/commands/index.ts index 790939fa..83998879 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -3,7 +3,8 @@ import { subcommands } from 'cmd-ts'; import { CliInfo } from '../cli.info.js'; import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; -import { importTopographicMaps } from './basemaps-topo-import/import-topographic-maps.js'; +import { topoCogCreation } from './basemaps-topo-import/topo-cog-creation.js'; +import { topoStacCreation } from './basemaps-topo-import/topo-stac-creation.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; import { commandGeneratePath } from './generate-path/path.generate.js'; @@ -49,7 +50,8 @@ export const AllCommands = { cmds: { 'create-pr': basemapsCreatePullRequest, 'create-mapsheet': basemapsCreateMapSheet, - 'import-topographic-maps': importTopographicMaps, + 'topo-stac-creation': topoStacCreation, + 'topo-cog-creation': topoCogCreation, }, }), 'pretty-print': commandPrettyPrint, From a283ab9fd022f727be0892c08b8d492b80cc6c6a Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 28 Nov 2024 16:01:07 +1300 Subject: [PATCH 12/47] Ouput to tmp for tile.json --- src/commands/basemaps-topo-import/topo-stac-creation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 0a2513c2..0e837414 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -30,7 +30,7 @@ const brokenTiffs = new Map(); */ export const topoStacCreation = command({ name: 'topo-stac-creation', - description: 'List input topographic files and run dgal to standardize and import into target.', + description: 'List input topographic files, create StacItems, and generate tiles for grouping.', version: CliInfo.version, args: { config, @@ -79,7 +79,7 @@ export const topoStacCreation = command({ latest.forEach((item) => paths.push(new URL(`${args.scale}-latest/${item.id}.json`, targetURL).href)); // write stac items into an JSON array - await fsa.write(tryParseUrl(`${args.target}/tiles.json`), JSON.stringify(paths, null, 2)); + await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/tiles.json`), JSON.stringify(paths, null, 2)); logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); }, From 342e6b2783275645c5ed19a49055cabaa0c9e8f7 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 28 Nov 2024 16:45:42 +1300 Subject: [PATCH 13/47] Check exist before download --- src/commands/basemaps-topo-import/topo-cog-creation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 3ac2872e..b863fab0 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -91,6 +91,7 @@ async function createCogs(input: URL, tmp: URL): Promise { const sourceUrl = tryParseUrl(source); const filePath = path.parse(sourceUrl.href); const fileName = filePath.base; + if (!(await fsa.exists(sourceUrl))) throw new Error('Source file not found'); const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); const inputPath = new URL(fileName, tmpFolder); logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); From c57dd2d19541e2c43dcf667389bda35e62f66a8d Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 28 Nov 2024 16:50:28 +1300 Subject: [PATCH 14/47] Remove the slice for test. --- src/commands/basemaps-topo-import/topo-stac-creation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 0e837414..286b2cbe 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -111,7 +111,7 @@ async function loadTiffsToCreateStacs( ): Promise<{ latest: StacItem[]; others: StacItem[] }> { // extract all file paths from the source directory and convert them into URL objects logger.info({ source }, 'LoadTiffs:Start'); - const files = (await fsa.toArray(fsa.list(source))).slice(0, 20); + const files = await fsa.toArray(fsa.list(source)); const tiffs = await loadTiffsFromPaths(files, Q); // we need to assign each tiff to a group based on its map code (e.g. AB01) @@ -335,7 +335,7 @@ function createStacCollection(title: string, imageryBound: BoundingBox, items: S temporal: { interval: [[cliDate, null]] }, }, links: items.map((item) => { - return { href: `./${item.id}.json`, rel: 'item', type: 'application/json', latest: true }; + return { href: `./${item.id}.json`, rel: 'item', type: 'application/json' }; }), }; From ef9d78aa4101a0ea56bd2be71e0e939b82e6d4f4 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Fri, 29 Nov 2024 13:43:08 +1300 Subject: [PATCH 15/47] refined Stac item.json and collection.json creation processes Co-authored-by: Wentao Kuang --- .../basemaps-topo-import/topo-cog-creation.ts | 2 +- .../topo-stac-creation.ts | 161 +++++++++++------- 2 files changed, 96 insertions(+), 67 deletions(-) diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index b863fab0..9b395b11 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -83,7 +83,7 @@ async function createCogs(input: URL, tmp: URL): Promise { try { // Extract the source URL from the item logger.info({ item: item.id }, 'CogCreation:Start'); - const source = item.links.find((l) => l.rel === 'linz_basemaps:source')?.href; + const source = item.assets['source']?.href; if (source == null) throw new Error('No source file found in the item'); await mkdir(tmpFolder, { recursive: true }); diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 286b2cbe..d751e6ff 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -13,7 +13,7 @@ import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; import { isArgo } from '../../utils/argo.js'; import { findBoundingBox } from '../../utils/geotiff.js'; -import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; +import { config, forceOutput, registerCli, tryParseUrl, UrlFolder, verbose } from '../common.js'; const Q = pLimit(10); const projection = Projection.get(Nztm2000QuadTms); @@ -42,12 +42,12 @@ export const topoStacCreation = command({ description: 'Imported imagery title', }), source: option({ - type: string, + type: UrlFolder, long: 'source', description: 'Location of the source files', }), target: option({ - type: string, + type: UrlFolder, long: 'target', description: 'Target location for the output files', }), @@ -62,12 +62,9 @@ export const topoStacCreation = command({ registerCli(this, args); logger.info('ListJobs:Start'); - const sourceURL = tryParseUrl(args.source); - const targetURL = tryParseUrl(args.target); - const { latest, others } = await loadTiffsToCreateStacs( - sourceURL, - targetURL, + args.source, + args.target, args.title, args.forceOutput, args.scale, @@ -75,8 +72,8 @@ export const topoStacCreation = command({ if (latest.length === 0 || others.length === 0) throw new Error('No Stac items created'); const paths: string[] = []; - others.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, targetURL).href)); - latest.forEach((item) => paths.push(new URL(`${args.scale}-latest/${item.id}.json`, targetURL).href)); + others.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, args.target).href)); + latest.forEach((item) => paths.push(new URL(`${args.scale}-latest/${item.id}.json`, args.target).href)); // write stac items into an JSON array await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/tiles.json`), JSON.stringify(paths, null, 2)); @@ -88,6 +85,7 @@ export const topoStacCreation = command({ interface VersionedTiff { version: string; tiff: Tiff; + bounds: Bounds; } /** @@ -130,12 +128,18 @@ async function loadTiffsToCreateStacs( const source = tiff.source.url.href; const { mapCode, version } = extractMapSheetNameWithVersion(source); + const bounds = await extractBounds(tiff); + if (bounds == null) { + brokenTiffs.set(`${mapCode}_${version}`, tiff); + continue; + } + const entry = versionsByMapCode.get(mapCode); if (entry == null) { - versionsByMapCode.set(mapCode, [{ version, tiff }]); + versionsByMapCode.set(mapCode, [{ version, tiff, bounds }]); } else { - entry.push({ version, tiff }); + entry.push({ version, tiff, bounds }); } } @@ -170,9 +174,9 @@ async function loadTiffsToCreateStacs( otherStacs.push(...stacItems.others); if (imageryBound == null) { - imageryBound = stacItems.bounds; + imageryBound = latest.bounds; } else { - imageryBound = imageryBound.union(stacItems.bounds); + imageryBound = imageryBound.union(latest.bounds); } } @@ -231,10 +235,10 @@ async function createStacItems( others: VersionedTiff[], target: URL, scale: string, -): Promise<{ latest: StacItem; others: StacItem[]; bounds: Bounds }> { - const latestStacItem = await createBaseStacItem(mapCode, mapCode, latest.version, latest.tiff); - const othersStacItems = await Promise.all( - [...others, latest].map(({ version, tiff }) => createBaseStacItem(`${mapCode}_${version}`, mapCode, version, tiff)), +): Promise<{ latest: StacItem; others: StacItem[] }> { + const latestStacItem = createBaseStacItem(mapCode, mapCode, latest.version, latest.tiff, latest.bounds); + const othersStacItems = [...others, latest].map(({ version, tiff, bounds }) => + createBaseStacItem(`${mapCode}_${version}`, mapCode, version, tiff, bounds), ); // need to do the part where they add special fields to each group @@ -250,69 +254,80 @@ async function createStacItems( }); // add link to latest referencing its copy that will live in topo50 dir - latestStacItem?.links.push({ + latestStacItem.links.push({ href: latestURL.href, rel: 'derived_from', type: 'application/json', }); - // since all of the given tiffs are for the same map code, - // their bounds should be the same, so we can grab the bounds of the latest - try { - const bounds = Bounds.fromBbox(await findBoundingBox(latest.tiff)); - - if (latestStacItem == null) { - throw new Error(`Error creating StacItem for latest version of ${mapCode} sheets`); - } - - return { latest: latestStacItem, others: othersStacItems.flatMap((item) => (item ? item : [])), bounds }; - } catch (e) { - throw new Error(`Couldn't extract bounds from the ${mapCode}'s latest map sheet`); - } + return { latest: latestStacItem, others: othersStacItems.flatMap((item) => (item ? item : [])) }; } -async function createBaseStacItem(id: string, mapCode: string, version: string, tiff: Tiff): Promise { - const source = tiff.source.url.href; - - let bounds; +/** + * This function attempts to extract bounds from the given Tiff object. + * + * @param tiff: The Tiff object from which to extract bounds + * + * @returns if succeeded, a Bounds object. Otherwise, null. + */ +async function extractBounds(tiff: Tiff): Promise { try { - bounds = Bounds.fromBbox(await findBoundingBox(tiff)); + return Bounds.fromBbox(await findBoundingBox(tiff)); } catch (e) { - brokenTiffs.set(id, tiff); return null; } +} +/** + * This function creates a base StacItem object based on the provided parameters. + * @param id: The id of the StacItem + * @example + * + * @param mapCode The map code of the map sheet + * @example "CJ10" + * + * @param version The version of the map sheet + * @example "v1-00" + * + * @param tiff TODO + * + * @param bounds TODO + * + * @returns + */ +function createBaseStacItem(id: string, mapCode: string, version: string, tiff: Tiff, bounds: Bounds): StacItem { logger.info({ id }, 'CreateStac:Item'); const item: StacItem = { - id: id, type: 'Feature', - collection: CliId, stac_version: '1.0.0', - stac_extensions: [], - geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, - bbox: projection.boundsToWgs84BoundingBox(bounds), + id: id, links: [ - { href: `./${id}.json`, rel: 'self' }, - { href: './collection.json', rel: 'collection' }, - { href: './collection.json', rel: 'parent' }, - { href: source, rel: 'linz_basemaps:source', type: 'image/tiff; application=geotiff' }, + { rel: 'self', href: `./${id}.json`, type: 'application/json' }, + { rel: 'collection', href: './collection.json', type: 'application/json' }, + { rel: 'parent', href: './collection.json', type: 'application/json' }, ], + assets: { + cog: { + href: `./${id}.tiff`, + type: 'image/tiff; application=geotiff; profile=cloud-optimized', + roles: ['data'], + }, + source: { + href: tiff.source.url.href, + type: 'image/tiff; application=geotiff', + roles: ['data'], + }, + }, + stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], properties: { - mapCode, - version, datetime: cliDate, - start_datetime: undefined, - end_datetime: undefined, + map_code: mapCode, // e.g. "CJ10" + version: version.replace('-', '.'), // convert from "v1-00" to "v1.00" 'proj:epsg': projection.epsg.code, - created_at: cliDate, - updated_at: cliDate, - 'linz:lifecycle': 'ongoing', - 'linz:geospatial_category': 'topographic-maps', - 'linz:region': 'new-zealand', - 'linz:security_classification': 'unclassified', - 'linz:slug': 'topo50', }, - assets: {}, + geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, + bbox: projection.boundsToWgs84BoundingBox(bounds), + collection: CliId, }; return item; @@ -321,22 +336,36 @@ async function createBaseStacItem(id: string, mapCode: string, version: string, function createStacCollection(title: string, imageryBound: BoundingBox, items: StacItem[]): StacCollection { logger.info({ items: items.length }, 'CreateStac:Collection'); const collection: StacCollection = { - id: CliId, type: 'Collection', stac_version: '1.0.0', - stac_extensions: [], - license: 'CC-BY-4.0', + id: CliId, title, description: 'Topographic maps of New Zealand', + license: 'CC-BY-4.0', + links: [ + // TODO: We not have an ODR bucket for the linz-topographic yet. + // { + // rel: 'root', + // href: 'https://nz-imagery.s3.ap-southeast-2.amazonaws.com/catalog.json', + // type: 'application/json', + // }, + { rel: 'self', href: './collection.json', type: 'application/json' }, + ...items.map((item) => { + return { href: `./${item.id}.json`, rel: 'item', type: 'application/json' }; + }), + ], providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], + 'linz:lifecycle': 'ongoing', + 'linz:geospatial_category': 'topographic-maps', + 'linz:region': 'new-zealand', + 'linz:security_classification': 'unclassified', + 'linz:slug': 'topo50', extent: { spatial: { bbox: [projection.boundsToWgs84BoundingBox(imageryBound)] }, - // Default the temporal time today if no times were found as it is required for STAC + // Default the temporal time today if no times were found as it is required for STAC temporal: { interval: [[cliDate, null]] }, }, - links: items.map((item) => { - return { href: `./${item.id}.json`, rel: 'item', type: 'application/json' }; - }), + stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], }; return collection; From 86a67118bc0e640bec9492b407ca384b0c85db4a Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Fri, 29 Nov 2024 15:30:36 +1300 Subject: [PATCH 16/47] added new task to append file:checksum info after cog creation Co-authored-by: Wentao Kuang --- .../basemaps-topo-import/topo-cog-creation.ts | 13 +++ .../topo-stac-creation.ts | 14 ++-- .../topo-stac-validation.ts | 83 +++++++++++++++++++ src/commands/index.ts | 2 + 4 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 src/commands/basemaps-topo-import/topo-stac-validation.ts diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 9b395b11..502029d0 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -1,6 +1,7 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; import { fsa } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; import { command, option, optional, restPositionals, string } from 'cmd-ts'; @@ -78,6 +79,7 @@ export const topoCogCreation = command({ }); async function createCogs(input: URL, tmp: URL): Promise { + const startTime = performance.now(); const item = await fsa.readJson(input); const tmpFolder = new URL(item.id, tmp); try { @@ -108,9 +110,20 @@ async function createCogs(input: URL, tmp: URL): Promise { const readStream = fsa.readStream(tempPath).pipe(new HashTransform('sha256')); const outputPath = tryParseUrl(input.href.replace('.json', '.tiff')); await fsa.write(outputPath, readStream); + + // Add the asset of created cog into stac item + const stac = await fsa.read(outputPath); + item.assets['cog'] = { + href: `./${item.id}.tiff`, + type: 'image/tiff; application=geotiff; profile=cloud-optimized', + roles: ['data'], + ...createFileStats(stac), + }; + await fsa.write(input, JSON.stringify(item, null, 2)); } finally { // Cleanup the temporary folder once everything is done logger.info({ path: tmpFolder.href }, 'CogCreation:Cleanup'); await rm(tmpFolder.href, { recursive: true, force: true }); + logger.info({ duration: performance.now() - startTime }, 'CogCreation:Done'); } } diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index d751e6ff..db72ad1c 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -281,7 +281,7 @@ async function extractBounds(tiff: Tiff): Promise { /** * This function creates a base StacItem object based on the provided parameters. * @param id: The id of the StacItem - * @example + * @example "CJ10" or "CJ10_v1-00" * * @param mapCode The map code of the map sheet * @example "CJ10" @@ -307,11 +307,6 @@ function createBaseStacItem(id: string, mapCode: string, version: string, tiff: { rel: 'parent', href: './collection.json', type: 'application/json' }, ], assets: { - cog: { - href: `./${id}.tiff`, - type: 'image/tiff; application=geotiff; profile=cloud-optimized', - roles: ['data'], - }, source: { href: tiff.source.url.href, type: 'image/tiff; application=geotiff', @@ -351,7 +346,12 @@ function createStacCollection(title: string, imageryBound: BoundingBox, items: S // }, { rel: 'self', href: './collection.json', type: 'application/json' }, ...items.map((item) => { - return { href: `./${item.id}.json`, rel: 'item', type: 'application/json' }; + return { + href: `./${item.id}.json`, + rel: 'item', + type: 'application/json', + // "file:checksum": "122061aa9d0283cda0a587d812a5c31a9cfb07c54e0f68f87f2d886675bf8a409709" + }; }), ], providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], diff --git a/src/commands/basemaps-topo-import/topo-stac-validation.ts b/src/commands/basemaps-topo-import/topo-stac-validation.ts new file mode 100644 index 00000000..1341cada --- /dev/null +++ b/src/commands/basemaps-topo-import/topo-stac-validation.ts @@ -0,0 +1,83 @@ +import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; +import { fsa } from '@basemaps/shared'; +import { command, option } from 'cmd-ts'; +import path from 'path'; +import { StacCollection } from 'stac-ts'; + +import { CliInfo } from '../../cli.info.js'; +import { logger } from '../../log.js'; +import { config, forceOutput, registerCli, UrlFolder, verbose } from '../common.js'; + +interface Stats { + 'file:size': number; + 'file:checksum': string; +} + +/** + * This command adds checksum properties to the StacItems an StacCollection + * JSON files that live in the input directory. + * + * @param input: Location of the StacItems and StacCollection to validate. + * @example s3://linz-topographic/maps/topo50/gridless_300dpi/2193/ + */ +export const topoStacValidation = command({ + name: 'topo-stac-validation', + description: 'Get the list of topo cog stac items and creating cog for them.', + version: CliInfo.version, + args: { + config, + verbose, + forceOutput, + input: option({ + type: UrlFolder, + long: 'input', + description: 'Path of stac files to validate', + }), + }, + async handler(args) { + const startTime = performance.now(); + registerCli(this, args); + logger.info('StacValidate:Start'); + const files = await fsa.toArray(fsa.list(args.input)); + const items = files.filter((f) => f.href.endsWith('json') && !f.href.endsWith('collection.json')); + if (items.length === 0) throw new Error('No item.json files found in the input path'); + + logger.info('StacValidate:LoadStacItems'); + const itemsMap = await getStatsFromFiles(items); + + const collection = files.find((f) => f.href.endsWith('collection.json')); + if (collection == null) throw new Error('No collection.json found in the input path'); + + logger.info('StacValidate:ValidateStac'); + const collectionStac = await fsa.readJson(collection); + const links = collectionStac.links; + for (const link of links) { + if (link.rel === 'item') { + const filename = link.href.replace('./', ''); + const itemStats = itemsMap.get(filename); + if (itemStats == null) throw new Error(`Stac item ${link.href} from collection is not found or duplicated`); + link['file:checksum'] = itemStats['file:checksum']; + itemsMap.delete(filename); + } + } + if (itemsMap.size > 0) throw new Error(`number: ${itemsMap.size} stac items not found from collection.`); + + // Write the stac collection after validation + logger.info('StacValidate:UpdateStac'); + await fsa.write(collection, JSON.stringify(collectionStac, null, 2)); + logger.info({ duration: performance.now() - startTime }, 'StacValidate:Done'); + }, +}); + +async function getStatsFromFiles(items: URL[]): Promise> { + const itemsMap = new Map(); + for (const item of items) { + const filePath = path.parse(item.href); + const buffer = await fsa.read(item); + const stats = createFileStats(buffer); + + itemsMap.set(filePath.base, stats); + } + + return itemsMap; +} diff --git a/src/commands/index.ts b/src/commands/index.ts index 83998879..d3bf5b50 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -5,6 +5,7 @@ import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; import { topoCogCreation } from './basemaps-topo-import/topo-cog-creation.js'; import { topoStacCreation } from './basemaps-topo-import/topo-stac-creation.js'; +import { topoStacValidation } from './basemaps-topo-import/topo-stac-validation.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; import { commandGeneratePath } from './generate-path/path.generate.js'; @@ -52,6 +53,7 @@ export const AllCommands = { 'create-mapsheet': basemapsCreateMapSheet, 'topo-stac-creation': topoStacCreation, 'topo-cog-creation': topoCogCreation, + 'topo-stac-validation': topoStacValidation, }, }), 'pretty-print': commandPrettyPrint, From edac1c18b96e2ba52583334ac60a8f0a94233bed Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Fri, 29 Nov 2024 16:03:38 +1300 Subject: [PATCH 17/47] added 'include-latest' flag for validation task (wip) Co-authored-by: Wentao Kuang --- .../topo-stac-validation.ts | 61 ++++++++++++------- 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/src/commands/basemaps-topo-import/topo-stac-validation.ts b/src/commands/basemaps-topo-import/topo-stac-validation.ts index 1341cada..c751cb98 100644 --- a/src/commands/basemaps-topo-import/topo-stac-validation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-validation.ts @@ -1,6 +1,6 @@ import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; import { fsa } from '@basemaps/shared'; -import { command, option } from 'cmd-ts'; +import { boolean, command, flag, option } from 'cmd-ts'; import path from 'path'; import { StacCollection } from 'stac-ts'; @@ -33,38 +33,53 @@ export const topoStacValidation = command({ long: 'input', description: 'Path of stac files to validate', }), + includeLatest: flag({ + type: boolean, + long: 'include-latest', + description: 'Include the validate for latest topo stacs', + }), }, async handler(args) { const startTime = performance.now(); registerCli(this, args); logger.info('StacValidate:Start'); - const files = await fsa.toArray(fsa.list(args.input)); - const items = files.filter((f) => f.href.endsWith('json') && !f.href.endsWith('collection.json')); - if (items.length === 0) throw new Error('No item.json files found in the input path'); + const inputs = [args.input]; + if (args.includeLatest) { + const base = args.input.href.endsWith('/') + ? args.input.href.slice(0, args.input.href.length - 1) + : args.input.href; + inputs.push(new URL(`${base}-latest/`)); + } + + for (const input of inputs) { + const files = await fsa.toArray(fsa.list(input)); + const items = files.filter((f) => f.href.endsWith('json') && !f.href.endsWith('collection.json')); + if (items.length === 0) throw new Error('No item.json files found in the input path'); - logger.info('StacValidate:LoadStacItems'); - const itemsMap = await getStatsFromFiles(items); + logger.info('StacValidate:LoadStacItems'); + const itemsMap = await getStatsFromFiles(items); - const collection = files.find((f) => f.href.endsWith('collection.json')); - if (collection == null) throw new Error('No collection.json found in the input path'); + const collection = files.find((f) => f.href.endsWith('collection.json')); + if (collection == null) throw new Error('No collection.json found in the input path'); - logger.info('StacValidate:ValidateStac'); - const collectionStac = await fsa.readJson(collection); - const links = collectionStac.links; - for (const link of links) { - if (link.rel === 'item') { - const filename = link.href.replace('./', ''); - const itemStats = itemsMap.get(filename); - if (itemStats == null) throw new Error(`Stac item ${link.href} from collection is not found or duplicated`); - link['file:checksum'] = itemStats['file:checksum']; - itemsMap.delete(filename); + logger.info('StacValidate:ValidateStac'); + const collectionStac = await fsa.readJson(collection); + const links = collectionStac.links; + for (const link of links) { + if (link.rel === 'item') { + const filename = link.href.replace('./', ''); + const itemStats = itemsMap.get(filename); + if (itemStats == null) throw new Error(`Stac item ${link.href} from collection is not found or duplicated`); + link['file:checksum'] = itemStats['file:checksum']; + itemsMap.delete(filename); + } } - } - if (itemsMap.size > 0) throw new Error(`number: ${itemsMap.size} stac items not found from collection.`); + if (itemsMap.size > 0) throw new Error(`number: ${itemsMap.size} stac items not found from collection.`); - // Write the stac collection after validation - logger.info('StacValidate:UpdateStac'); - await fsa.write(collection, JSON.stringify(collectionStac, null, 2)); + // Write the stac collection after validation + logger.info('StacValidate:UpdateStac'); + await fsa.write(collection, JSON.stringify(collectionStac, null, 2)); + } logger.info({ duration: performance.now() - startTime }, 'StacValidate:Done'); }, }); From 63e2a58697c4f7de4300f1f5999134667124f885 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Tue, 3 Dec 2024 09:37:13 +1300 Subject: [PATCH 18/47] refined the topo-stac-creation processes and moved its functions into seperate files --- .../__test__/import-topographic-maps.test.ts | 6 +- .../extractors/extract-bounds.ts | 19 + .../extract-map-code-and-version.ts | 33 ++ .../mappers/group-by-map-code.ts | 93 +++++ .../stac/create-base-stac-item.ts | 60 ++++ .../stac/create-stac-collection.ts | 51 +++ .../stac/create-stac-item-groups.ts | 46 +++ .../stac/write-stac-files.ts | 28 ++ .../topo-stac-creation.ts | 334 +++--------------- 9 files changed, 375 insertions(+), 295 deletions(-) create mode 100644 src/commands/basemaps-topo-import/extractors/extract-bounds.ts create mode 100644 src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts create mode 100644 src/commands/basemaps-topo-import/mappers/group-by-map-code.ts create mode 100644 src/commands/basemaps-topo-import/stac/create-base-stac-item.ts create mode 100644 src/commands/basemaps-topo-import/stac/create-stac-collection.ts create mode 100644 src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts create mode 100644 src/commands/basemaps-topo-import/stac/write-stac-files.ts diff --git a/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts index 3710dc44..75346a95 100644 --- a/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts +++ b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { describe, it } from 'node:test'; -import { extractMapSheetNameWithVersion } from '../topo-stac-creation.js'; +import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; describe('extractMapSheetName', () => { const FakeDomain = 's3://topographic/fake-domain'; @@ -22,7 +22,7 @@ describe('extractMapSheetName', () => { it('Should parse the correct MapSheet Names', async () => { for (const file of FakeFiles) { - const output = extractMapSheetNameWithVersion(file.input); + const output = extractMapCodeAndVersion(file.input); assert.equal(output.mapCode, file.expected.mapCode, 'Map code does not match'); assert.equal(output.version, file.expected.version, 'Version does not match'); } @@ -31,7 +31,7 @@ describe('extractMapSheetName', () => { it('Should not able to parse a version from file', async () => { const wrongFiles = [`${FakeDomain}/MB07_GeoTif1-00.tif`, `${FakeDomain}/MB07_TIFF_600v1.tif`]; for (const file of wrongFiles) { - assert.throws(() => extractMapSheetNameWithVersion(file), new Error('Version not found in the file name')); + assert.throws(() => extractMapCodeAndVersion(file), new Error('Version not found in the file name')); } }); }); diff --git a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts new file mode 100644 index 00000000..1b833731 --- /dev/null +++ b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts @@ -0,0 +1,19 @@ +import { Bounds } from '@basemaps/geo'; +import { Tiff } from '@cogeotiff/core'; + +import { findBoundingBox } from '../../../utils/geotiff.js'; + +/** + * This function attempts to extract bounds from the given Tiff object. + * + * @param tiff: The Tiff object from which to extract bounds + * + * @returns if succeeded, a Bounds object. Otherwise, null. + */ +export async function extractBounds(tiff: Tiff): Promise { + try { + return Bounds.fromBbox(await findBoundingBox(tiff)); + } catch (e) { + return null; + } +} diff --git a/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts b/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts new file mode 100644 index 00000000..31e7e78c --- /dev/null +++ b/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts @@ -0,0 +1,33 @@ +import path from 'path'; + +import { logger } from '../../../log.js'; +import { tryParseUrl } from '../../common.js'; + +/** + * Extract the map code and version from the provided path. + * Throws an error if either detail cannot be parsed. + * + * @param file: filepath from which to extract the map code and version + * + * @example + * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" + * returns: { mapCode: "CJ10", version: "v1-00" } + * + * @returns an object containing the map code and version + */ +export function extractMapCodeAndVersion(file: string): { mapCode: string; version: string } { + const url = tryParseUrl(file); + const filePath = path.parse(url.href); + const fileName = filePath.name; + + // extract map code from head of the file name (e.g. CJ10) + const mapCode = fileName.split('_')[0]; + if (mapCode == null) throw new Error('Map sheet not found in the file name'); + + // extract version from tail of the file name (e.g. v1-0) + const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; + if (version == null) throw new Error('Version not found in the file name'); + + logger.info({ mapCode, version }, 'extractMapCodeAndVersion()'); + return { mapCode, version }; +} diff --git a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts new file mode 100644 index 00000000..d2a3a244 --- /dev/null +++ b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts @@ -0,0 +1,93 @@ +import { Bounds } from '@basemaps/geo'; +import { Tiff } from '@cogeotiff/core'; + +import { extractBounds } from '../extractors/extract-bounds.js'; +import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; +import { brokenTiffs } from '../topo-stac-creation.js'; + +export interface VersionedTiff { + version: string; + tiff: Tiff; + bounds: Bounds; +} + +type VersionsByMapCode = Map; +export type GroupsByMapCode = Map; + +/** + * We need to assign each tiff to a group based on its map code (e.g. "AT24"). + * For each group, we then need to identify the latest version and set it aside from the rest. + * The latest version will have special metadata, whereas the rest will have similar metadata. + * + * @param tiffs: The tiffs to group by map code and version + * @returns a `GroupsByMapCode` Map object + */ +export async function groupTiffsByMapCodeAndLatest(tiffs: Tiff[]): Promise { + // group the tiffs by map code and version + // + // { + // "AT24": [ + // { version: "v1-00", tiff: Tiff }, + // { version: "v1-01", tiff: Tiff }, + // ... + // ], + // "AT25": [ + // { version: "v1-00", tiff: Tiff }, + // { version: "v2-00", tiff: Tiff }, + // ... + // ] + // } + const versionsByMapCode: VersionsByMapCode = new Map(); + + for (const tiff of tiffs) { + const source = tiff.source.url.href; + const { mapCode, version } = extractMapCodeAndVersion(source); + + const bounds = await extractBounds(tiff); + if (bounds == null) { + brokenTiffs.set(`${mapCode}_${version}`, tiff); + continue; + } + + const entry = versionsByMapCode.get(mapCode); + + if (entry == null) { + versionsByMapCode.set(mapCode, [{ version, tiff, bounds }]); + } else { + entry.push({ version, tiff, bounds }); + } + } + + // for each group, identify the latest version + // + // { + // "AT24": { + // latest: { version: "v1-01", tiff: Tiff }, + // others: [ + // { version: "v1-00", tiff: Tiff }, + // ... + // ] + // }, + // "AT25": { + // latest: { version: "v2-00", tiff: Tiff }, + // others: [ + // { version: "v1-00", tiff: Tiff }, + // ... + // ] + // } + // } + const groupsByMapCode: GroupsByMapCode = new Map(); + + for (const [mapCode, versions] of versionsByMapCode.entries()) { + const sorted = versions.sort((a, b) => a.version.localeCompare(b.version)); + + const latest = sorted[sorted.length - 1]; + if (latest == null) throw new Error(); + + const others = sorted.filter((version) => version !== latest); + + groupsByMapCode.set(mapCode, { latest, others }); + } + + return groupsByMapCode; +} diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts new file mode 100644 index 00000000..88880c26 --- /dev/null +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -0,0 +1,60 @@ +import { Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { Tiff } from '@cogeotiff/core'; +import { StacItem } from 'stac-ts'; +import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; + +import { logger } from '../../../log.js'; + +const projection = Projection.get(Nztm2000QuadTms); +const cliDate = new Date().toISOString(); + +/** + * This function creates a base StacItem object based on the provided parameters. + * @param id: The id of the StacItem + * @example "CJ10" or "CJ10_v1-00" + * + * @param mapCode The map code of the map sheet + * @example "CJ10" + * + * @param version The version of the map sheet + * @example "v1-00" + * + * @param tiff TODO + * + * @param bounds TODO + * + * @returns + */ +export function createBaseStacItem(id: string, mapCode: string, version: string, tiff: Tiff, bounds: Bounds): StacItem { + logger.info({ id }, 'createBaseStacItem()'); + const item: StacItem = { + type: 'Feature', + stac_version: '1.0.0', + id: id, + links: [ + { rel: 'self', href: `./${id}.json`, type: 'application/json' }, + { rel: 'collection', href: './collection.json', type: 'application/json' }, + { rel: 'parent', href: './collection.json', type: 'application/json' }, + ], + assets: { + source: { + href: tiff.source.url.href, + type: 'image/tiff; application=geotiff', + roles: ['data'], + }, + }, + stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], + properties: { + datetime: cliDate, + map_code: mapCode, // e.g. "CJ10" + version: version.replace('-', '.'), // convert from "v1-00" to "v1.00" + 'proj:epsg': projection.epsg.code, + }, + geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, + bbox: projection.boundsToWgs84BoundingBox(bounds), + collection: CliId, + }; + + return item; +} diff --git a/src/commands/basemaps-topo-import/stac/create-stac-collection.ts b/src/commands/basemaps-topo-import/stac/create-stac-collection.ts new file mode 100644 index 00000000..92846b5a --- /dev/null +++ b/src/commands/basemaps-topo-import/stac/create-stac-collection.ts @@ -0,0 +1,51 @@ +import { BoundingBox, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { StacCollection, StacItem } from 'stac-ts'; + +import { logger } from '../../../log.js'; + +const projection = Projection.get(Nztm2000QuadTms); +const cliDate = new Date().toISOString(); + +export function createStacCollection(title: string, imageryBound: BoundingBox, items: StacItem[]): StacCollection { + logger.info({ items: items.length }, 'CreateStacCollection()'); + const collection: StacCollection = { + type: 'Collection', + stac_version: '1.0.0', + id: CliId, + title, + description: 'Topographic maps of New Zealand', + license: 'CC-BY-4.0', + links: [ + // TODO: We not have an ODR bucket for the linz-topographic yet. + // { + // rel: 'root', + // href: 'https://nz-imagery.s3.ap-southeast-2.amazonaws.com/catalog.json', + // type: 'application/json', + // }, + { rel: 'self', href: './collection.json', type: 'application/json' }, + ...items.map((item) => { + return { + href: `./${item.id}.json`, + rel: 'item', + type: 'application/json', + // "file:checksum": "122061aa9d0283cda0a587d812a5c31a9cfb07c54e0f68f87f2d886675bf8a409709" + }; + }), + ], + providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], + 'linz:lifecycle': 'ongoing', + 'linz:geospatial_category': 'topographic-maps', + 'linz:region': 'new-zealand', + 'linz:security_classification': 'unclassified', + 'linz:slug': 'topo50', + extent: { + spatial: { bbox: [projection.boundsToWgs84BoundingBox(imageryBound)] }, + // Default the temporal time today if no times were found as it is required for STAC + temporal: { interval: [[cliDate, null]] }, + }, + stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], + }; + + return collection; +} diff --git a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts new file mode 100644 index 00000000..8798a26b --- /dev/null +++ b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts @@ -0,0 +1,46 @@ +import { StacItem } from 'stac-ts'; + +import { VersionedTiff } from '../mappers/group-by-map-code.js'; +import { createBaseStacItem } from './create-base-stac-item.js'; + +/** + * This function needs to create two groups: + * - StacItem objects that will live in the "topo[50/250]" directory + * - StacItem objects that will live in the "topo[50/250]-latest" directory + * + * All versions need a StacItem object that lives in the topo[50/250] directory + * The latest version needs a second StacItem object that lives in the topo[50/250]-latest dir + */ +export async function createStacItemGroups( + mapCode: string, + latest: VersionedTiff, + others: VersionedTiff[], + target: URL, + scale: string, +): Promise<{ latest: StacItem; all: StacItem[] }> { + const latestStacItem = createBaseStacItem(mapCode, mapCode, latest.version, latest.tiff, latest.bounds); + const allStacItems = [...others, latest].map(({ version, tiff, bounds }) => + createBaseStacItem(`${mapCode}_${version}`, mapCode, version, tiff, bounds), + ); + + // need to do the part where they add special fields to each group + const latestURL = new URL(`${scale}/${mapCode}_${latest.version}.json`, target); + + // add link to all items pointing to the latest version + allStacItems.forEach((item) => { + item?.links.push({ + href: latestURL.href, + rel: 'latest-version', + type: 'application/json', + }); + }); + + // add link to the latest item referencing its copy that will live in the topo[50/250] directory + latestStacItem.links.push({ + href: latestURL.href, + rel: 'derived_from', + type: 'application/json', + }); + + return { latest: latestStacItem, all: allStacItems }; +} diff --git a/src/commands/basemaps-topo-import/stac/write-stac-files.ts b/src/commands/basemaps-topo-import/stac/write-stac-files.ts new file mode 100644 index 00000000..b612f69b --- /dev/null +++ b/src/commands/basemaps-topo-import/stac/write-stac-files.ts @@ -0,0 +1,28 @@ +import { fsa } from '@basemaps/shared'; +import { StacCollection, StacItem } from 'stac-ts'; + +import { logger } from '../../../log.js'; +import { isArgo } from '../../../utils/argo.js'; +import { brokenTiffs } from '../topo-stac-creation.js'; + +export async function writeStacFiles( + target: URL, + force: boolean, + items: StacItem[], + collection: StacCollection, +): Promise { + // Create collection json for all topo50-latest items. + if (force || isArgo()) { + logger.info({ target }, 'CreateStac:Output'); + logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); + for (const item of items) { + const itemPath = new URL(`${item.id}.json`, target); + await fsa.write(itemPath, JSON.stringify(item, null, 2)); + } + const collectionPath = new URL('collection.json', target); + await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); + } + + const brokenPath = new URL('/tmp/topo-stac-creation/output/broken.json', target); + await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); +} diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index db72ad1c..40ae9f19 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -1,24 +1,21 @@ import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; -import { BoundingBox, Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { Bounds } from '@basemaps/geo'; import { fsa } from '@basemaps/shared'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; import { Tiff } from '@cogeotiff/core'; import { command, option, string } from 'cmd-ts'; import pLimit from 'p-limit'; -import path from 'path'; -import { StacCollection, StacItem } from 'stac-ts'; -import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; +import { StacItem } from 'stac-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; -import { isArgo } from '../../utils/argo.js'; -import { findBoundingBox } from '../../utils/geotiff.js'; import { config, forceOutput, registerCli, tryParseUrl, UrlFolder, verbose } from '../common.js'; +import { groupTiffsByMapCodeAndLatest } from './mappers/group-by-map-code.js'; +import { createStacCollection } from './stac/create-stac-collection.js'; +import { createStacItemGroups } from './stac/create-stac-item-groups.js'; +import { writeStacFiles } from './stac/write-stac-files.js'; const Q = pLimit(10); -const projection = Projection.get(Nztm2000QuadTms); -const cliDate = new Date().toISOString(); -const brokenTiffs = new Map(); +export const brokenTiffs = new Map(); /** * List all the tiffs in a directory for topographic maps and create cogs for each. @@ -62,17 +59,17 @@ export const topoStacCreation = command({ registerCli(this, args); logger.info('ListJobs:Start'); - const { latest, others } = await loadTiffsToCreateStacs( + const { latest, all } = await loadTiffsToCreateStacs( args.source, args.target, args.title, args.forceOutput, args.scale, ); - if (latest.length === 0 || others.length === 0) throw new Error('No Stac items created'); + if (latest.length === 0 || all.length === 0) throw new Error('No Stac items created'); const paths: string[] = []; - others.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, args.target).href)); + all.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, args.target).href)); latest.forEach((item) => paths.push(new URL(`${args.scale}-latest/${item.id}.json`, args.target).href)); // write stac items into an JSON array @@ -82,12 +79,6 @@ export const topoStacCreation = command({ }, }); -interface VersionedTiff { - version: string; - tiff: Tiff; - bounds: Bounds; -} - /** * @param source: Source directory URL from which to load tiff files * @example TODO @@ -106,289 +97,48 @@ async function loadTiffsToCreateStacs( title: string, force: boolean, scale: string, -): Promise<{ latest: StacItem[]; others: StacItem[] }> { - // extract all file paths from the source directory and convert them into URL objects +): Promise<{ latest: StacItem[]; all: StacItem[] }> { logger.info({ source }, 'LoadTiffs:Start'); - const files = await fsa.toArray(fsa.list(source)); - const tiffs = await loadTiffsFromPaths(files, Q); - - // we need to assign each tiff to a group based on its map code (e.g. AB01) - // for each group, we then need to identify the latest version and set it aside from the rest - // the latest version will have special metadata, whereas the rest will have similar metadata - - // group the tiffs by map code - // - // { - // "AB01": ["v1-00", "v1-01", "v2-00"] - // "CD01": ["v1-00", "v2-00", "v2-01"] - // } - const versionsByMapCode: Map = new Map(); - - for (const tiff of tiffs) { - const source = tiff.source.url.href; - const { mapCode, version } = extractMapSheetNameWithVersion(source); - - const bounds = await extractBounds(tiff); - if (bounds == null) { - brokenTiffs.set(`${mapCode}_${version}`, tiff); - continue; - } - - const entry = versionsByMapCode.get(mapCode); - - if (entry == null) { - versionsByMapCode.set(mapCode, [{ version, tiff, bounds }]); - } else { - entry.push({ version, tiff, bounds }); - } - } - - // for each group, identify the latest version - // - // { - // "AB01": { latest: "v2-00", others: ["v1-00", "v1-01"] } - // "CD01": { latest: "v2-01", others: ["v1-00", "v2-00"] } - // } - const groupsByMapCode: Map = new Map(); - - for (const [mapCode, versions] of versionsByMapCode.entries()) { - const sorted = versions.sort((a, b) => a.version.localeCompare(b.version)); - - const latest = sorted[sorted.length - 1]; - if (latest == null) throw new Error(); - - const others = sorted.filter((version) => version !== latest); + // extract all file paths from the source directory and convert them into URL objects + const fileURLs = await fsa.toArray(fsa.list(source)); + // process all of the URL objects into Tiff objects + const tiffs = await loadTiffsFromPaths(fileURLs.slice(0, 10), Q); + logger.info({ numTiffs: tiffs.length }, 'LoadTiffs:End'); - groupsByMapCode.set(mapCode, { latest, others }); - } + logger.info('GroupTiffs:Start'); + // group all of the Tiff objects by map code, version, and latest + const groupsByMapCode = await groupTiffsByMapCodeAndLatest(tiffs); + logger.info({ numGroups: groupsByMapCode.size }, 'GroupTiffs:End'); - const latestStacs: StacItem[] = []; - const otherStacs: StacItem[] = []; - let imageryBound: Bounds | undefined; + logger.info('CreateStacItems:Start'); + const latestBounds: Bounds[] = []; + const allBounds: Bounds[] = []; - logger.info({ tiffs: tiffs.length }, 'CreateStac:Start'); + const latestItems: StacItem[] = []; + const allItems: StacItem[] = []; for (const [mapCode, { latest, others }] of groupsByMapCode.entries()) { - const stacItems = await createStacItems(mapCode, latest, others, target, scale); - latestStacs.push(stacItems.latest); - otherStacs.push(...stacItems.others); + // push bounds to their respective arrays + latestBounds.push(latest.bounds); + allBounds.push(...[latest, ...others].map((vt) => vt.bounds)); - if (imageryBound == null) { - imageryBound = latest.bounds; - } else { - imageryBound = imageryBound.union(latest.bounds); - } - } - - if (imageryBound == null) throw new Error('No imagery bounds found'); - - // Create collection json for all topo50 items - const latestCollection = createStacCollection(title, imageryBound, latestStacs); - const collection = createStacCollection(title, imageryBound, otherStacs); - - await writeStacFiles(new URL(`${scale}-latest/`, target), force, latestStacs, latestCollection); - await writeStacFiles(new URL(`${scale}/`, target), force, otherStacs, collection); - - return { latest: latestStacs, others: otherStacs }; -} - -/** - * Extract the map code and version from the provided path. - * Throws an error if either detail cannot be parsed. - * - * @param file: filepath from which to extract the map code and version - * - * @example - * file: "s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/CJ10_GRIDLESS_GeoTifv1-00.tif" - * returns: { mapCode: "CJ10", version: "v1-00" } - * - * @returns an object containing the map code and version - */ -export function extractMapSheetNameWithVersion(file: string): { mapCode: string; version: string } { - const url = tryParseUrl(file); - const filePath = path.parse(url.href); - const fileName = filePath.name; - - // extract map code from head of the file name (e.g. CJ10) - const mapCode = fileName.split('_')[0]; - if (mapCode == null) throw new Error('Map sheet not found in the file name'); - - // extract version from tail of the file name (e.g. v1-0) - const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; - if (version == null) throw new Error('Version not found in the file name'); - - logger.info({ mapCode, version }, 'ListJobs:Output'); - return { mapCode, version }; -} + // create StacItem object groups + const stacItems = await createStacItemGroups(mapCode, latest, others, target, scale); -/** - * This function needs to create two groups: - * - StacItem objects that will live in the Topo50 directory - * - StacItem objects that will live in the Topo250 directory - * - * All versions need a StacItem object that lives in the Topo50 dir - * The latest version needs a second StacItem object that lives in the Topo250 dir - */ -async function createStacItems( - mapCode: string, - latest: VersionedTiff, - others: VersionedTiff[], - target: URL, - scale: string, -): Promise<{ latest: StacItem; others: StacItem[] }> { - const latestStacItem = createBaseStacItem(mapCode, mapCode, latest.version, latest.tiff, latest.bounds); - const othersStacItems = [...others, latest].map(({ version, tiff, bounds }) => - createBaseStacItem(`${mapCode}_${version}`, mapCode, version, tiff, bounds), - ); - - // need to do the part where they add special fields to each group - const latestURL = new URL(`${scale}/${mapCode}_${latest.version}.json`, target); - - // add link to others pointing to latest - othersStacItems.forEach((item) => { - item?.links.push({ - href: latestURL.href, - rel: 'latest-version', - type: 'application/json', - }); - }); - - // add link to latest referencing its copy that will live in topo50 dir - latestStacItem.links.push({ - href: latestURL.href, - rel: 'derived_from', - type: 'application/json', - }); - - return { latest: latestStacItem, others: othersStacItems.flatMap((item) => (item ? item : [])) }; -} - -/** - * This function attempts to extract bounds from the given Tiff object. - * - * @param tiff: The Tiff object from which to extract bounds - * - * @returns if succeeded, a Bounds object. Otherwise, null. - */ -async function extractBounds(tiff: Tiff): Promise { - try { - return Bounds.fromBbox(await findBoundingBox(tiff)); - } catch (e) { - return null; + // push StacItem objects to their respective arrays + latestItems.push(stacItems.latest); + allItems.push(...stacItems.all); } -} + logger.info({ numLatestItems: latestItems.length, numItems: allItems.length }, 'CreateStacItems:End'); -/** - * This function creates a base StacItem object based on the provided parameters. - * @param id: The id of the StacItem - * @example "CJ10" or "CJ10_v1-00" - * - * @param mapCode The map code of the map sheet - * @example "CJ10" - * - * @param version The version of the map sheet - * @example "v1-00" - * - * @param tiff TODO - * - * @param bounds TODO - * - * @returns - */ -function createBaseStacItem(id: string, mapCode: string, version: string, tiff: Tiff, bounds: Bounds): StacItem { - logger.info({ id }, 'CreateStac:Item'); - const item: StacItem = { - type: 'Feature', - stac_version: '1.0.0', - id: id, - links: [ - { rel: 'self', href: `./${id}.json`, type: 'application/json' }, - { rel: 'collection', href: './collection.json', type: 'application/json' }, - { rel: 'parent', href: './collection.json', type: 'application/json' }, - ], - assets: { - source: { - href: tiff.source.url.href, - type: 'image/tiff; application=geotiff', - roles: ['data'], - }, - }, - stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], - properties: { - datetime: cliDate, - map_code: mapCode, // e.g. "CJ10" - version: version.replace('-', '.'), // convert from "v1-00" to "v1.00" - 'proj:epsg': projection.epsg.code, - }, - geometry: projection.boundsToGeoJsonFeature(bounds).geometry as GeoJSONPolygon, - bbox: projection.boundsToWgs84BoundingBox(bounds), - collection: CliId, - }; - - return item; -} - -function createStacCollection(title: string, imageryBound: BoundingBox, items: StacItem[]): StacCollection { - logger.info({ items: items.length }, 'CreateStac:Collection'); - const collection: StacCollection = { - type: 'Collection', - stac_version: '1.0.0', - id: CliId, - title, - description: 'Topographic maps of New Zealand', - license: 'CC-BY-4.0', - links: [ - // TODO: We not have an ODR bucket for the linz-topographic yet. - // { - // rel: 'root', - // href: 'https://nz-imagery.s3.ap-southeast-2.amazonaws.com/catalog.json', - // type: 'application/json', - // }, - { rel: 'self', href: './collection.json', type: 'application/json' }, - ...items.map((item) => { - return { - href: `./${item.id}.json`, - rel: 'item', - type: 'application/json', - // "file:checksum": "122061aa9d0283cda0a587d812a5c31a9cfb07c54e0f68f87f2d886675bf8a409709" - }; - }), - ], - providers: [{ name: 'Land Information New Zealand', roles: ['host', 'licensor', 'processor', 'producer'] }], - 'linz:lifecycle': 'ongoing', - 'linz:geospatial_category': 'topographic-maps', - 'linz:region': 'new-zealand', - 'linz:security_classification': 'unclassified', - 'linz:slug': 'topo50', - extent: { - spatial: { bbox: [projection.boundsToWgs84BoundingBox(imageryBound)] }, - // Default the temporal time today if no times were found as it is required for STAC - temporal: { interval: [[cliDate, null]] }, - }, - stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], - }; - - return collection; -} + // Create collection json for all topo50 items + logger.info('CreateStacCollections:Start'); + const latestCollection = createStacCollection(title, Bounds.union(latestBounds), latestItems); + const nonLatestCollection = createStacCollection(title, Bounds.union(allBounds), allItems); -async function writeStacFiles( - target: URL, - force: boolean, - items: StacItem[], - collection: StacCollection, -): Promise { - // Create collection json for all topo50-latest items. - if (force || isArgo()) { - logger.info({ target }, 'CreateStac:Output'); - logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); - for (const item of items) { - const itemPath = new URL(`${item.id}.json`, target); - await fsa.write(itemPath, JSON.stringify(item, null, 2)); - } - const collectionPath = new URL('collection.json', target); - await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); - } + await writeStacFiles(new URL(`${scale}-latest/`, target), force, latestItems, latestCollection); + await writeStacFiles(new URL(`${scale}/`, target), force, allItems, nonLatestCollection); + logger.info({ tiffs: tiffs.length }, 'CreateStacCollections:End'); - const brokenPath = new URL('/tmp/topo-stac-creation/output/broken.json', target); - await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); + return { latest: latestItems, all: allItems }; } From 8fa54f327c80cc39d44bd55ffc8708ad492e095f Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 3 Dec 2024 11:31:59 +1300 Subject: [PATCH 19/47] Add alpha layer for the created cogs --- src/commands/basemaps-topo-import/gdal-commands.ts | 11 ++++++++++- .../basemaps-topo-import/topo-cog-creation.ts | 11 ++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts index fac1275c..be4630e7 100644 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ b/src/commands/basemaps-topo-import/gdal-commands.ts @@ -2,6 +2,15 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; import { urlToString } from '../common.js'; +export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { + if (source.length === 0) throw new Error('No source files given for :' + targetVrt.href); + return { + output: targetVrt, + command: 'gdalbuildvrt', + args: ['-addalpha', urlToString(targetVrt), ...source.map(urlToString)], + }; +} + export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { return { command: 'gdal_translate', @@ -11,7 +20,7 @@ export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { ['-of', 'COG'], // Output format ['-stats'], // Force stats (re)computation ['-a_srs', `EPSG:2193`], // Projection override - // creation options (-co) + ['-co', 'ADD_ALPHA=YES'], ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes ['-co', 'BIGTIFF=NO'], diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 502029d0..cf831272 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -15,7 +15,7 @@ import { logger } from '../../log.js'; import { HashTransform } from '../../utils/hash.stream.js'; import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; import { loadInput } from '../group/group.js'; -import { gdalBuildCogCommands } from './gdal-commands.js'; +import { gdalBuildCogCommands, gdalBuildVrt } from './gdal-commands.js'; const Q = pLimit(10); /** @@ -99,11 +99,16 @@ async function createCogs(input: URL, tmp: URL): Promise { logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); await fsa.write(inputPath, hashStreamSource); + // run gdal_buildvrt for the source file + const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); + const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath]); + await new GdalRunner(commandBuildVrt).run(logger); + // run gdal_translate for each job logger.info({ item: item.id }, 'CogCreation:gdal_translate'); const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const command = gdalBuildCogCommands(inputPath, tempPath); - await new GdalRunner(command).run(logger); + const commandTranslate = gdalBuildCogCommands(vrtPath, tempPath); + await new GdalRunner(commandTranslate).run(logger); // fsa.write output to target location logger.info({ item: item.id }, 'CogCreation:Output'); From d676df2f991bce18a500eb8a1e1b9587992ac852 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Tue, 3 Dec 2024 15:08:25 +1300 Subject: [PATCH 20/47] updated the StacItem creation process to extract the epsg from each Tiff file --- .../__test__/import-topographic-maps.test.ts | 2 +- .../extractors/extract-epsg-from-tiff.ts | 30 +++++++++++++++++++ .../stac/create-base-stac-item.ts | 9 ++++-- .../topo-stac-creation.ts | 2 +- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts diff --git a/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts index 75346a95..f3ae8574 100644 --- a/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts +++ b/src/commands/basemaps-topo-import/__test__/import-topographic-maps.test.ts @@ -3,7 +3,7 @@ import { describe, it } from 'node:test'; import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; -describe('extractMapSheetName', () => { +describe('extractMapCodeAndVersion', () => { const FakeDomain = 's3://topographic/fake-domain'; const FakeFiles = [ { input: `${FakeDomain}/MB07_GeoTifv1-00.tif`, expected: { mapCode: 'MB07', version: 'v1-00' } }, diff --git a/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts b/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts new file mode 100644 index 00000000..c282f65b --- /dev/null +++ b/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts @@ -0,0 +1,30 @@ +import { Epsg } from '@basemaps/geo'; +import { Tiff, TiffTagGeo } from '@cogeotiff/core'; + +import { logger } from '../../../log.js'; +import { extractEpsg } from '../../generate-path/path.generate.js'; + +export function extractEpsgFromTiff(tiff: Tiff): Epsg { + try { + const epsg = Epsg.tryGet(extractEpsg(tiff)); + if (epsg != null) return epsg; + } catch { + logger.warn('Could not extract epsg code directly from tiff'); + } + + const tag = tiff.images[0]?.valueGeo(TiffTagGeo.ProjectedCitationGeoKey); + + if (tag?.startsWith('Universal Transverse Mercator Zone')) { + return Epsg.Wgs84; + } + + if (tag?.startsWith('Chatham Islands Transverse Mercator 2000')) { + return Epsg.Citm2000; + } + + if (tag?.startsWith('New Zealand Transverse Mercator 2000')) { + return Epsg.Nztm2000; + } + + throw new Error('Could not extract epsg code'); +} diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts index 88880c26..34fd95d0 100644 --- a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -1,12 +1,12 @@ -import { Bounds, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { Bounds, Projection } from '@basemaps/geo'; import { CliId } from '@basemaps/shared/build/cli/info.js'; import { Tiff } from '@cogeotiff/core'; import { StacItem } from 'stac-ts'; import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; import { logger } from '../../../log.js'; +import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; -const projection = Projection.get(Nztm2000QuadTms); const cliDate = new Date().toISOString(); /** @@ -28,6 +28,11 @@ const cliDate = new Date().toISOString(); */ export function createBaseStacItem(id: string, mapCode: string, version: string, tiff: Tiff, bounds: Bounds): StacItem { logger.info({ id }, 'createBaseStacItem()'); + + const epsg = extractEpsgFromTiff(tiff); + const projection = Projection.tryGet(epsg); + if (projection == null) throw new Error(`Could not find a projection for epsg:${epsg.code}`); + const item: StacItem = { type: 'Feature', stac_version: '1.0.0', diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 40ae9f19..af19a189 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -102,7 +102,7 @@ async function loadTiffsToCreateStacs( // extract all file paths from the source directory and convert them into URL objects const fileURLs = await fsa.toArray(fsa.list(source)); // process all of the URL objects into Tiff objects - const tiffs = await loadTiffsFromPaths(fileURLs.slice(0, 10), Q); + const tiffs = await loadTiffsFromPaths(fileURLs, Q); logger.info({ numTiffs: tiffs.length }, 'LoadTiffs:End'); logger.info('GroupTiffs:Start'); From 11869354e7d920b90752a1a9f22e85d54361cb77 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Tue, 3 Dec 2024 16:31:36 +1300 Subject: [PATCH 21/47] Reproject chatham islands --- .../basemaps-topo-import/gdal-commands.ts | 22 +++++++++++++- .../mappers/group-by-map-code.ts | 30 ++++++++++++++++--- .../stac/create-base-stac-item.ts | 24 +++++++-------- .../stac/create-stac-collection.ts | 5 +++- .../stac/create-stac-item-groups.ts | 14 ++++----- .../basemaps-topo-import/topo-cog-creation.ts | 19 ++++++++---- .../topo-stac-creation.ts | 2 +- 7 files changed, 82 insertions(+), 34 deletions(-) diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts index be4630e7..b40a0a97 100644 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ b/src/commands/basemaps-topo-import/gdal-commands.ts @@ -1,4 +1,5 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { Epsg, Nztm2000QuadTms } from '@basemaps/geo'; import { urlToString } from '../common.js'; @@ -11,6 +12,25 @@ export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { }; } +export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { + return { + output: targetVrt, + command: 'gdalwarp', + args: [ + ['-of', 'vrt'], // Output as a VRT + '-multi', // Mutithread IO + ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp + ['-s_srs', sourceProj.toEpsgString()], // Source EPSG + ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG + urlToString(sourceVrt), + urlToString(targetVrt), + ] + .filter((f) => f != null) + .flat() + .map(String), + }; +} + export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { return { command: 'gdal_translate', @@ -19,7 +39,7 @@ export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { ['-q'], // Supress non-error output ['-of', 'COG'], // Output format ['-stats'], // Force stats (re)computation - ['-a_srs', `EPSG:2193`], // Projection override + ['-a_srs', Nztm2000QuadTms.projection.toEpsgString()], // Projection override ['-co', 'ADD_ALPHA=YES'], ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes diff --git a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts index d2a3a244..d3009a95 100644 --- a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts +++ b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts @@ -1,14 +1,25 @@ -import { Bounds } from '@basemaps/geo'; +import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; +import { Bounds, Epsg, Projection } from '@basemaps/geo'; +import { fsa } from '@basemaps/shared'; import { Tiff } from '@cogeotiff/core'; import { extractBounds } from '../extractors/extract-bounds.js'; +import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; import { brokenTiffs } from '../topo-stac-creation.js'; +export interface FileStats { + 'file:size': number; + 'file:checksum': string; +} + export interface VersionedTiff { version: string; tiff: Tiff; + stats: FileStats; + epsg: Epsg; bounds: Bounds; + source: string; } type VersionsByMapCode = Map; @@ -40,6 +51,11 @@ export async function groupTiffsByMapCodeAndLatest(tiffs: Tiff[]): Promise { + const stats = createFileStats(JSON.stringify(item, null, 2)); return { href: `./${item.id}.json`, rel: 'item', type: 'application/json', - // "file:checksum": "122061aa9d0283cda0a587d812a5c31a9cfb07c54e0f68f87f2d886675bf8a409709" + 'file:checksum': stats['file:checksum'], + 'file:size': stats['file:size'], }; }), ], diff --git a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts index 8798a26b..b30f059a 100644 --- a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts +++ b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts @@ -15,21 +15,17 @@ export async function createStacItemGroups( mapCode: string, latest: VersionedTiff, others: VersionedTiff[], - target: URL, scale: string, ): Promise<{ latest: StacItem; all: StacItem[] }> { - const latestStacItem = createBaseStacItem(mapCode, mapCode, latest.version, latest.tiff, latest.bounds); - const allStacItems = [...others, latest].map(({ version, tiff, bounds }) => - createBaseStacItem(`${mapCode}_${version}`, mapCode, version, tiff, bounds), + const latestStacItem = createBaseStacItem(mapCode, mapCode, latest); + const allStacItems = [...others, latest].map((versionedTiff) => + createBaseStacItem(`${mapCode}_${versionedTiff.version}`, mapCode, versionedTiff), ); - // need to do the part where they add special fields to each group - const latestURL = new URL(`${scale}/${mapCode}_${latest.version}.json`, target); - // add link to all items pointing to the latest version allStacItems.forEach((item) => { item?.links.push({ - href: latestURL.href, + href: `./${mapCode}_${latest.version}.json`, rel: 'latest-version', type: 'application/json', }); @@ -37,7 +33,7 @@ export async function createStacItemGroups( // add link to the latest item referencing its copy that will live in the topo[50/250] directory latestStacItem.links.push({ - href: latestURL.href, + href: `../${scale}/${mapCode}_${latest.version}.json`, rel: 'derived_from', type: 'application/json', }); diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index cf831272..b3d59f5e 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -2,6 +2,7 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; +import { Epsg } from '@basemaps/geo'; import { fsa } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; import { command, option, optional, restPositionals, string } from 'cmd-ts'; @@ -15,7 +16,7 @@ import { logger } from '../../log.js'; import { HashTransform } from '../../utils/hash.stream.js'; import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; import { loadInput } from '../group/group.js'; -import { gdalBuildCogCommands, gdalBuildVrt } from './gdal-commands.js'; +import { gdalBuildCogCommands, gdalBuildVrt, gdalBuildVrtWarp } from './gdal-commands.js'; const Q = pLimit(10); /** @@ -99,15 +100,23 @@ async function createCogs(input: URL, tmp: URL): Promise { logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); await fsa.write(inputPath, hashStreamSource); - // run gdal_buildvrt for the source file + // run gdal commands for each the source file + logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath]); await new GdalRunner(commandBuildVrt).run(logger); - // run gdal_translate for each job - logger.info({ item: item.id }, 'CogCreation:gdal_translate'); + logger.info({ item: item.id }, 'CogCreation:gdalwarp'); const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const commandTranslate = gdalBuildCogCommands(vrtPath, tempPath); + const sourceEpsg = Number(item.properties['source:epsg']); + const sourceProj = Epsg.tryGet(sourceEpsg); + if (sourceProj == null) throw new Error(`Unknown source projection ${sourceEpsg}`); + const vrtWarpPath = new URL(`${item.id}-warp.vrt`, tmpFolder); + const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj); + await new GdalRunner(commandBuildVrtWarp).run(logger); + + logger.info({ item: item.id }, 'CogCreation:gdal_translate'); + const commandTranslate = gdalBuildCogCommands(vrtWarpPath, tempPath); await new GdalRunner(commandTranslate).run(logger); // fsa.write output to target location diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index af19a189..b464682c 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -123,7 +123,7 @@ async function loadTiffsToCreateStacs( allBounds.push(...[latest, ...others].map((vt) => vt.bounds)); // create StacItem object groups - const stacItems = await createStacItemGroups(mapCode, latest, others, target, scale); + const stacItems = await createStacItemGroups(mapCode, latest, others, scale); // push StacItem objects to their respective arrays latestItems.push(stacItems.latest); From 5ac61abc336cfe9bea3c377c0df3256bed094c46 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 4 Dec 2024 10:42:41 +1300 Subject: [PATCH 22/47] Add checksum for source in cog creation. --- .../basemaps-topo-import/mappers/group-by-map-code.ts | 11 ++--------- .../stac/create-base-stac-item.ts | 2 -- .../basemaps-topo-import/topo-cog-creation.ts | 4 ++++ 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts index d3009a95..5857de91 100644 --- a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts +++ b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts @@ -1,6 +1,4 @@ -import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; import { Bounds, Epsg, Projection } from '@basemaps/geo'; -import { fsa } from '@basemaps/shared'; import { Tiff } from '@cogeotiff/core'; import { extractBounds } from '../extractors/extract-bounds.js'; @@ -16,7 +14,6 @@ export interface FileStats { export interface VersionedTiff { version: string; tiff: Tiff; - stats: FileStats; epsg: Epsg; bounds: Bounds; source: string; @@ -66,17 +63,13 @@ export async function groupTiffsByMapCodeAndLatest(tiffs: Tiff[]): Promise { const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); const inputPath = new URL(fileName, tmpFolder); logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); + // Add checksum for source file + if (item.assets['source'] == null) throw new Error('No source file found in the item'); + item.assets['source']['file:checksum'] = hashStreamSource.multihash; + item.assets['source']['file:size'] = hashStreamSource.size; await fsa.write(inputPath, hashStreamSource); // run gdal commands for each the source file From ec3ce26cc888d5efd5b272c6a1969a9ba90a56cf Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 4 Dec 2024 11:03:36 +1300 Subject: [PATCH 23/47] Stop thrown for broken epsg tiles --- .../extractors/extract-epsg-from-tiff.ts | 4 +- .../mappers/group-by-map-code.ts | 17 +++- .../basemaps-topo-import/topo-cog-creation.ts | 2 +- .../topo-stac-validation.ts | 98 ------------------- src/commands/index.ts | 2 - 5 files changed, 15 insertions(+), 108 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/topo-stac-validation.ts diff --git a/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts b/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts index c282f65b..d3d09b2c 100644 --- a/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts +++ b/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts @@ -4,7 +4,7 @@ import { Tiff, TiffTagGeo } from '@cogeotiff/core'; import { logger } from '../../../log.js'; import { extractEpsg } from '../../generate-path/path.generate.js'; -export function extractEpsgFromTiff(tiff: Tiff): Epsg { +export function extractEpsgFromTiff(tiff: Tiff): Epsg | null { try { const epsg = Epsg.tryGet(extractEpsg(tiff)); if (epsg != null) return epsg; @@ -26,5 +26,5 @@ export function extractEpsgFromTiff(tiff: Tiff): Epsg { return Epsg.Nztm2000; } - throw new Error('Could not extract epsg code'); + return null; } diff --git a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts index 5857de91..7dc67987 100644 --- a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts +++ b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts @@ -1,6 +1,7 @@ import { Bounds, Epsg, Projection } from '@basemaps/geo'; import { Tiff } from '@cogeotiff/core'; +import { logger } from '../../../log.js'; import { extractBounds } from '../extractors/extract-bounds.js'; import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; @@ -48,21 +49,27 @@ export async function groupTiffsByMapCodeAndLatest(tiffs: Tiff[]): Promise { logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); // Add checksum for source file if (item.assets['source'] == null) throw new Error('No source file found in the item'); + await fsa.write(inputPath, hashStreamSource); item.assets['source']['file:checksum'] = hashStreamSource.multihash; item.assets['source']['file:size'] = hashStreamSource.size; - await fsa.write(inputPath, hashStreamSource); // run gdal commands for each the source file logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); diff --git a/src/commands/basemaps-topo-import/topo-stac-validation.ts b/src/commands/basemaps-topo-import/topo-stac-validation.ts deleted file mode 100644 index c751cb98..00000000 --- a/src/commands/basemaps-topo-import/topo-stac-validation.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; -import { fsa } from '@basemaps/shared'; -import { boolean, command, flag, option } from 'cmd-ts'; -import path from 'path'; -import { StacCollection } from 'stac-ts'; - -import { CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { config, forceOutput, registerCli, UrlFolder, verbose } from '../common.js'; - -interface Stats { - 'file:size': number; - 'file:checksum': string; -} - -/** - * This command adds checksum properties to the StacItems an StacCollection - * JSON files that live in the input directory. - * - * @param input: Location of the StacItems and StacCollection to validate. - * @example s3://linz-topographic/maps/topo50/gridless_300dpi/2193/ - */ -export const topoStacValidation = command({ - name: 'topo-stac-validation', - description: 'Get the list of topo cog stac items and creating cog for them.', - version: CliInfo.version, - args: { - config, - verbose, - forceOutput, - input: option({ - type: UrlFolder, - long: 'input', - description: 'Path of stac files to validate', - }), - includeLatest: flag({ - type: boolean, - long: 'include-latest', - description: 'Include the validate for latest topo stacs', - }), - }, - async handler(args) { - const startTime = performance.now(); - registerCli(this, args); - logger.info('StacValidate:Start'); - const inputs = [args.input]; - if (args.includeLatest) { - const base = args.input.href.endsWith('/') - ? args.input.href.slice(0, args.input.href.length - 1) - : args.input.href; - inputs.push(new URL(`${base}-latest/`)); - } - - for (const input of inputs) { - const files = await fsa.toArray(fsa.list(input)); - const items = files.filter((f) => f.href.endsWith('json') && !f.href.endsWith('collection.json')); - if (items.length === 0) throw new Error('No item.json files found in the input path'); - - logger.info('StacValidate:LoadStacItems'); - const itemsMap = await getStatsFromFiles(items); - - const collection = files.find((f) => f.href.endsWith('collection.json')); - if (collection == null) throw new Error('No collection.json found in the input path'); - - logger.info('StacValidate:ValidateStac'); - const collectionStac = await fsa.readJson(collection); - const links = collectionStac.links; - for (const link of links) { - if (link.rel === 'item') { - const filename = link.href.replace('./', ''); - const itemStats = itemsMap.get(filename); - if (itemStats == null) throw new Error(`Stac item ${link.href} from collection is not found or duplicated`); - link['file:checksum'] = itemStats['file:checksum']; - itemsMap.delete(filename); - } - } - if (itemsMap.size > 0) throw new Error(`number: ${itemsMap.size} stac items not found from collection.`); - - // Write the stac collection after validation - logger.info('StacValidate:UpdateStac'); - await fsa.write(collection, JSON.stringify(collectionStac, null, 2)); - } - logger.info({ duration: performance.now() - startTime }, 'StacValidate:Done'); - }, -}); - -async function getStatsFromFiles(items: URL[]): Promise> { - const itemsMap = new Map(); - for (const item of items) { - const filePath = path.parse(item.href); - const buffer = await fsa.read(item); - const stats = createFileStats(buffer); - - itemsMap.set(filePath.base, stats); - } - - return itemsMap; -} diff --git a/src/commands/index.ts b/src/commands/index.ts index d3bf5b50..83998879 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -5,7 +5,6 @@ import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; import { topoCogCreation } from './basemaps-topo-import/topo-cog-creation.js'; import { topoStacCreation } from './basemaps-topo-import/topo-stac-creation.js'; -import { topoStacValidation } from './basemaps-topo-import/topo-stac-validation.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; import { commandGeneratePath } from './generate-path/path.generate.js'; @@ -53,7 +52,6 @@ export const AllCommands = { 'create-mapsheet': basemapsCreateMapSheet, 'topo-stac-creation': topoStacCreation, 'topo-cog-creation': topoCogCreation, - 'topo-stac-validation': topoStacValidation, }, }), 'pretty-print': commandPrettyPrint, From d8028e72abe6cd95b15d7405b61dd0a03e5f8063 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 4 Dec 2024 11:07:39 +1300 Subject: [PATCH 24/47] Write the broken file out --- src/commands/basemaps-topo-import/topo-stac-creation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index b464682c..bd575fe2 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -74,6 +74,7 @@ export const topoStacCreation = command({ // write stac items into an JSON array await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/tiles.json`), JSON.stringify(paths, null, 2)); + await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/broken.json`), JSON.stringify(brokenTiffs, null, 2)); logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); }, From 6c8c790c0d7c8fe69e16ad5962f3454862feceed Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 4 Dec 2024 11:41:14 +1300 Subject: [PATCH 25/47] Set same target resolution for NZTM --- src/commands/basemaps-topo-import/gdal-commands.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts index b40a0a97..cb688c77 100644 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ b/src/commands/basemaps-topo-import/gdal-commands.ts @@ -13,6 +13,7 @@ export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { } export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { + const targetResolution = Nztm2000QuadTms.pixelScale(0); return { output: targetVrt, command: 'gdalwarp', @@ -22,6 +23,7 @@ export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Eps ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp ['-s_srs', sourceProj.toEpsgString()], // Source EPSG ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG + ['-tr', targetResolution, targetResolution], urlToString(sourceVrt), urlToString(targetVrt), ] From 5677c04637de08b8e87f77d1523e061a41cca095 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Wed, 4 Dec 2024 14:16:12 +1300 Subject: [PATCH 26/47] updated cog creation process to match gsd based on projection --- src/commands/basemaps-topo-import/gdal-commands.ts | 12 +++++++++--- .../basemaps-topo-import/topo-cog-creation.ts | 10 ++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts index cb688c77..bc4c7e72 100644 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ b/src/commands/basemaps-topo-import/gdal-commands.ts @@ -1,5 +1,5 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { Epsg, Nztm2000QuadTms } from '@basemaps/geo'; +import { Epsg, Nztm2000QuadTms, Projection } from '@basemaps/geo'; import { urlToString } from '../common.js'; @@ -12,8 +12,14 @@ export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { }; } -export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { - const targetResolution = Nztm2000QuadTms.pixelScale(0); +export function gdalBuildVrtWarp( + targetVrt: URL, + sourceVrt: URL, + sourceProj: Epsg, + sourceRes: [number, number, number], +): GdalCommand { + const resZoom = Projection.getTiffResZoom(Nztm2000QuadTms, sourceRes[0]); + const targetResolution = Nztm2000QuadTms.pixelScale(resZoom); return { output: targetVrt, command: 'gdalwarp', diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 71d2ef5b..209ef67b 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -3,7 +3,7 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; import { Epsg } from '@basemaps/geo'; -import { fsa } from '@basemaps/shared'; +import { fsa, Tiff } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; import { command, option, optional, restPositionals, string } from 'cmd-ts'; import { mkdir, rm } from 'fs/promises'; @@ -96,6 +96,7 @@ async function createCogs(input: URL, tmp: URL): Promise { const fileName = filePath.base; if (!(await fsa.exists(sourceUrl))) throw new Error('Source file not found'); const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); + const inputPath = new URL(fileName, tmpFolder); logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); // Add checksum for source file @@ -104,6 +105,11 @@ async function createCogs(input: URL, tmp: URL): Promise { item.assets['source']['file:checksum'] = hashStreamSource.multihash; item.assets['source']['file:size'] = hashStreamSource.size; + // read resolution from first image of tiff + const tiff = await new Tiff(fsa.source(inputPath)).init(); + const sourceRes = tiff.images[0]?.resolution; + if (sourceRes == null) throw new Error('Could not read resolution from first image'); + // run gdal commands for each the source file logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); @@ -116,7 +122,7 @@ async function createCogs(input: URL, tmp: URL): Promise { const sourceProj = Epsg.tryGet(sourceEpsg); if (sourceProj == null) throw new Error(`Unknown source projection ${sourceEpsg}`); const vrtWarpPath = new URL(`${item.id}-warp.vrt`, tmpFolder); - const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj); + const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj, sourceRes); await new GdalRunner(commandBuildVrtWarp).run(logger); logger.info({ item: item.id }, 'CogCreation:gdal_translate'); From 659fd613e95e3518d3225bcdc7bc204db4acaca6 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 4 Dec 2024 14:25:43 +1300 Subject: [PATCH 27/47] Remove checksum for collection json item links --- .../basemaps-topo-import/mappers/group-by-map-code.ts | 6 +++--- .../basemaps-topo-import/stac/create-base-stac-item.ts | 2 +- .../basemaps-topo-import/stac/create-stac-collection.ts | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts index 7dc67987..d331a53e 100644 --- a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts +++ b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts @@ -17,7 +17,7 @@ export interface VersionedTiff { tiff: Tiff; epsg: Epsg; bounds: Bounds; - source: string; + source: URL; } type VersionsByMapCode = Map; @@ -49,8 +49,8 @@ export async function groupTiffsByMapCodeAndLatest(tiffs: Tiff[]): Promise { - const stats = createFileStats(JSON.stringify(item, null, 2)); return { href: `./${item.id}.json`, rel: 'item', type: 'application/json', - 'file:checksum': stats['file:checksum'], - 'file:size': stats['file:size'], + // 'file:checksum': stats['file:checksum'], + // 'file:size': stats['file:size'], }; }), ], From b19e45da0e0af09b0bfddbbd098b4bc69231878d Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Mon, 9 Dec 2024 10:56:15 +1300 Subject: [PATCH 28/47] refactor: updated processes to be epsg agnostic --- .../extractors/extract-bounds.ts | 7 +- .../extractors/extract-epsg-from-tiff.ts | 37 +++--- .../basemaps-topo-import/gdal-commands.ts | 50 ++++---- .../mappers/group-by-map-code.ts | 115 ------------------ .../mappers/group-tiffs-by-directory.ts | 79 ++++++++++++ .../stac/create-base-stac-item.ts | 14 +-- .../stac/create-stac-item-groups.ts | 27 ++-- .../stac/write-stac-files.ts | 4 - .../basemaps-topo-import/topo-cog-creation.ts | 3 +- .../topo-stac-creation.ts | 112 ++++++++++++----- .../types/by-directory.ts | 110 +++++++++++++++++ .../basemaps-topo-import/types/tiff-item.ts | 24 ++++ 12 files changed, 369 insertions(+), 213 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/mappers/group-by-map-code.ts create mode 100644 src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts create mode 100644 src/commands/basemaps-topo-import/types/by-directory.ts create mode 100644 src/commands/basemaps-topo-import/types/tiff-item.ts diff --git a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts index 1b833731..9ab361a8 100644 --- a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts +++ b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts @@ -1,6 +1,7 @@ import { Bounds } from '@basemaps/geo'; import { Tiff } from '@cogeotiff/core'; +import { logger } from '../../../log.js'; import { findBoundingBox } from '../../../utils/geotiff.js'; /** @@ -12,8 +13,12 @@ import { findBoundingBox } from '../../../utils/geotiff.js'; */ export async function extractBounds(tiff: Tiff): Promise { try { - return Bounds.fromBbox(await findBoundingBox(tiff)); + const bounds = Bounds.fromBbox(await findBoundingBox(tiff)); + + logger.info({ found: true }, 'extractBounds()'); + return bounds; } catch (e) { + logger.info({ found: false }, 'extractBounds()'); return null; } } diff --git a/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts b/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts index d3d09b2c..a40a11db 100644 --- a/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts +++ b/src/commands/basemaps-topo-import/extractors/extract-epsg-from-tiff.ts @@ -4,27 +4,32 @@ import { Tiff, TiffTagGeo } from '@cogeotiff/core'; import { logger } from '../../../log.js'; import { extractEpsg } from '../../generate-path/path.generate.js'; +const projections = [ + ['Universal Transverse Mercator Zone', Epsg.Wgs84], + ['Chatham Islands Transverse Mercator 2000', Epsg.Citm2000], + ['New Zealand Transverse Mercator 2000', Epsg.Nztm2000], +] as const; + export function extractEpsgFromTiff(tiff: Tiff): Epsg | null { + // try to extract the epsg directly from the tiff try { - const epsg = Epsg.tryGet(extractEpsg(tiff)); - if (epsg != null) return epsg; + const epsg = Epsg.get(extractEpsg(tiff)); + if (epsg != null) { + logger.info({ found: epsg.code }, 'extractEpsgFromTiff()'); + return epsg; + } } catch { - logger.warn('Could not extract epsg code directly from tiff'); - } - - const tag = tiff.images[0]?.valueGeo(TiffTagGeo.ProjectedCitationGeoKey); - - if (tag?.startsWith('Universal Transverse Mercator Zone')) { - return Epsg.Wgs84; - } - - if (tag?.startsWith('Chatham Islands Transverse Mercator 2000')) { - return Epsg.Citm2000; - } + // try to extract the epsg from the tiff's projected citation geotag + const tag = tiff.images[0]?.valueGeo(TiffTagGeo.ProjectedCitationGeoKey); - if (tag?.startsWith('New Zealand Transverse Mercator 2000')) { - return Epsg.Nztm2000; + for (const [citation, epsg] of projections) { + if (tag?.startsWith(citation)) { + logger.info({ found: epsg.code }, 'extractEpsgFromTiff()'); + return epsg; + } + } } + logger.info({ found: false }, 'extractEpsgFromTiff()'); return null; } diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts index bc4c7e72..70e225ad 100644 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ b/src/commands/basemaps-topo-import/gdal-commands.ts @@ -1,5 +1,5 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { Epsg, Nztm2000QuadTms, Projection } from '@basemaps/geo'; +import { Epsg } from '@basemaps/geo'; import { urlToString } from '../common.js'; @@ -8,28 +8,24 @@ export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { return { output: targetVrt, command: 'gdalbuildvrt', - args: ['-addalpha', urlToString(targetVrt), ...source.map(urlToString)], + args: [['-addalpha'], urlToString(targetVrt), ...source.map(urlToString)] + .filter((f) => f != null) + .flat() + .map(String), }; } -export function gdalBuildVrtWarp( - targetVrt: URL, - sourceVrt: URL, - sourceProj: Epsg, - sourceRes: [number, number, number], -): GdalCommand { - const resZoom = Projection.getTiffResZoom(Nztm2000QuadTms, sourceRes[0]); - const targetResolution = Nztm2000QuadTms.pixelScale(resZoom); +export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { return { output: targetVrt, command: 'gdalwarp', args: [ + ['-multi'], // Mutithread IO ['-of', 'vrt'], // Output as a VRT - '-multi', // Mutithread IO ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp ['-s_srs', sourceProj.toEpsgString()], // Source EPSG - ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG - ['-tr', targetResolution, targetResolution], + // ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG + // ['-tr', targetResolution, targetResolution], urlToString(sourceVrt), urlToString(targetVrt), ] @@ -41,24 +37,28 @@ export function gdalBuildVrtWarp( export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { return { - command: 'gdal_translate', output, + command: 'gdal_translate', args: [ ['-q'], // Supress non-error output - ['-of', 'COG'], // Output format ['-stats'], // Force stats (re)computation - ['-a_srs', Nztm2000QuadTms.projection.toEpsgString()], // Projection override - ['-co', 'ADD_ALPHA=YES'], - ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS - ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes + ['-of', 'COG'], // Output format + + // https://gdal.org/en/latest/drivers/raster/cog.html#creation-options ['-co', 'BIGTIFF=NO'], + ['-co', 'BLOCKSIZE=512'], + ['-co', 'COMPRESS=WEBP'], + ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS + ['-co', 'OVERVIEW_COMPRESS=WEBP'], ['-co', 'OVERVIEWS=IGNORE_EXISTING'], - ['-co', `BLOCKSIZE=512`], - ['-co', `COMPRESS=webp`], - ['-co', `QUALITY=100`], - ['-co', `OVERVIEW_COMPRESS=webp`], - ['-co', `OVERVIEW_RESAMPLING=lanczos`], - ['-co', `OVERVIEW_QUALITY=90`], + ['-co', 'OVERVIEW_QUALITY=90'], + ['-co', 'OVERVIEW_RESAMPLING=LANCZOS'], + ['-co', 'QUALITY=100'], + ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes + + // https://gdal.org/en/latest/drivers/raster/cog.html#reprojection-related-creation-options + ['-co', 'ADD_ALPHA=YES'], + urlToString(input), urlToString(output), ] diff --git a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts b/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts deleted file mode 100644 index d331a53e..00000000 --- a/src/commands/basemaps-topo-import/mappers/group-by-map-code.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { Bounds, Epsg, Projection } from '@basemaps/geo'; -import { Tiff } from '@cogeotiff/core'; - -import { logger } from '../../../log.js'; -import { extractBounds } from '../extractors/extract-bounds.js'; -import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; -import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; -import { brokenTiffs } from '../topo-stac-creation.js'; - -export interface FileStats { - 'file:size': number; - 'file:checksum': string; -} - -export interface VersionedTiff { - version: string; - tiff: Tiff; - epsg: Epsg; - bounds: Bounds; - source: URL; -} - -type VersionsByMapCode = Map; -export type GroupsByMapCode = Map; - -/** - * We need to assign each tiff to a group based on its map code (e.g. "AT24"). - * For each group, we then need to identify the latest version and set it aside from the rest. - * The latest version will have special metadata, whereas the rest will have similar metadata. - * - * @param tiffs: The tiffs to group by map code and version - * @returns a `GroupsByMapCode` Map object - */ -export async function groupTiffsByMapCodeAndLatest(tiffs: Tiff[]): Promise { - // group the tiffs by map code and version - // - // { - // "AT24": [ - // { version: "v1-00", tiff: Tiff }, - // { version: "v1-01", tiff: Tiff }, - // ... - // ], - // "AT25": [ - // { version: "v1-00", tiff: Tiff }, - // { version: "v2-00", tiff: Tiff }, - // ... - // ] - // } - const versionsByMapCode: VersionsByMapCode = new Map(); - - for (const tiff of tiffs) { - const source = tiff.source.url; - const { mapCode, version } = extractMapCodeAndVersion(source.href); - - const bounds = await extractBounds(tiff); - if (bounds == null) { - brokenTiffs.set(`${mapCode}_${version}`, tiff); - logger.warn({ mapCode, version }, 'Could not extract bounds from tiff'); - continue; - } - const entry = versionsByMapCode.get(mapCode); - - // extract the epsg code from the Tiff object - const epsg = extractEpsgFromTiff(tiff); - if (epsg == null) { - brokenTiffs.set(`${mapCode}_${version}`, tiff); - logger.warn({ mapCode, version }, 'Could not extract epsg from tiff'); - continue; - } - const projection = Projection.tryGet(epsg); - if (projection == null) throw new Error(`Could not find a projection for epsg:${epsg.code}`); - - // Convert bounds to WGS84 for different source epsg - const boundsCoverted = Bounds.fromBbox(projection.boundsToWgs84BoundingBox(bounds)); - - if (entry == null) { - versionsByMapCode.set(mapCode, [{ version, tiff, bounds: boundsCoverted, epsg, source }]); - } else { - entry.push({ version, tiff, bounds: boundsCoverted, epsg, source }); - } - } - - // for each group, identify the latest version - // - // { - // "AT24": { - // latest: { version: "v1-01", tiff: Tiff }, - // others: [ - // { version: "v1-00", tiff: Tiff }, - // ... - // ] - // }, - // "AT25": { - // latest: { version: "v2-00", tiff: Tiff }, - // others: [ - // { version: "v1-00", tiff: Tiff }, - // ... - // ] - // } - // } - const groupsByMapCode: GroupsByMapCode = new Map(); - - for (const [mapCode, versions] of versionsByMapCode.entries()) { - const sorted = versions.sort((a, b) => a.version.localeCompare(b.version)); - - const latest = sorted[sorted.length - 1]; - if (latest == null) throw new Error(); - - const others = sorted.filter((version) => version !== latest); - - groupsByMapCode.set(mapCode, { latest, others }); - } - - return groupsByMapCode; -} diff --git a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts new file mode 100644 index 00000000..c5ded1b5 --- /dev/null +++ b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts @@ -0,0 +1,79 @@ +import { Bounds, Projection } from '@basemaps/geo'; +import { Tiff } from '@cogeotiff/core'; + +import { logger } from '../../../log.js'; +import { extractBounds } from '../extractors/extract-bounds.js'; +import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; +import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; +import { brokenTiffs } from '../topo-stac-creation.js'; +import { ByDirectory } from '../types/by-directory.js'; +import { TiffItem } from '../types/tiff-item.js'; + +/** + * We need to assign each tiff to a group based on its map code (e.g. "AT24"). + * For each group, we then need to identify the latest version and set it aside from the rest. + * The latest version will have special metadata, whereas the rest will have similar metadata. + * + * @param tiffs: The tiffs to group by map code and version + * @returns a `SplitEpsgMap` Map object + */ +export async function groupTiffsByDirectory(tiffs: Tiff[]): Promise> { + // group the tiffs by directory, epsg, and map code + const byDirectory = new ByDirectory(); + + // create items for each tiff and store them into 'all' by {epsg} and {map code} + for (const tiff of tiffs) { + const source = tiff.source.url; + const { mapCode, version } = extractMapCodeAndVersion(source.href); + + const bounds = await extractBounds(tiff); + const epsg = extractEpsgFromTiff(tiff); + + if (bounds == null || epsg == null) { + if (bounds == null) { + brokenTiffs.noBounds.push(`${mapCode}_${version}`); + logger.warn({ mapCode, version }, 'Could not extract bounds from tiff'); + } + + if (epsg == null) { + brokenTiffs.noEpsg.push(`${mapCode}_${version}`); + logger.warn({ mapCode, version }, 'Could not extract epsg from tiff'); + } + + continue; + } + + const projection = Projection.tryGet(epsg); + if (projection == null) throw new Error(`Could not find a projection for epsg:${epsg.code}`); + + // Convert bounds to WGS84 for different source epsg + const boundsCoverted = Bounds.fromBbox(projection.boundsToWgs84BoundingBox(bounds)); + + const item = new TiffItem(tiff, source, mapCode, version, boundsCoverted, epsg); + + // push the item into 'all' by {epsg} and {map code} + byDirectory.all.get(epsg.toString()).get(mapCode, []).push(item); + } + + // for each {epsg} and {map code}, identify the latest item by {version} and copy it to 'latest' + for (const [epsg, byMapCode] of byDirectory.all.entries()) { + for (const [mapCode, items] of byMapCode.entries()) { + const sortedItems = items.sort((a, b) => a.version.localeCompare(b.version)); + + const latestItem = sortedItems[sortedItems.length - 1]; + if (latestItem == null) throw new Error(); + + // store the item into 'latest' by {epsg} and {map code} + byDirectory.latest.get(epsg).set(mapCode, latestItem); + } + } + + logger.info( + byDirectory.all.entries().reduce((obj, [epsg, byMapCode]) => { + return { ...obj, [epsg]: byMapCode.entries().length }; + }, {}), + 'numItemsPerEpsg', + ); + + return byDirectory; +} diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts index 8b973704..f565fe57 100644 --- a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -4,7 +4,7 @@ import { StacItem } from 'stac-ts'; import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; import { logger } from '../../../log.js'; -import { VersionedTiff } from '../mappers/group-by-map-code.js'; +import { TiffItem } from '../types/tiff-item.js'; const cliDate = new Date().toISOString(); @@ -25,7 +25,7 @@ const cliDate = new Date().toISOString(); * * @returns */ -export function createBaseStacItem(id: string, mapCode: string, versionedTiff: VersionedTiff): StacItem { +export function createBaseStacItem(id: string, mapCode: string, tiffItem: TiffItem): StacItem { logger.info({ id }, 'createBaseStacItem()'); const item: StacItem = { @@ -39,7 +39,7 @@ export function createBaseStacItem(id: string, mapCode: string, versionedTiff: V ], assets: { source: { - href: versionedTiff.source.href, + href: tiffItem.source.href, type: 'image/tiff; application=geotiff', roles: ['data'], }, @@ -48,12 +48,12 @@ export function createBaseStacItem(id: string, mapCode: string, versionedTiff: V properties: { datetime: cliDate, map_code: mapCode, // e.g. "CJ10" - version: versionedTiff.version.replace('-', '.'), // convert from "v1-00" to "v1.00" + version: tiffItem.version.replace('-', '.'), // convert from "v1-00" to "v1.00" 'proj:epsg': Epsg.Nztm2000.code, - 'source:epsg': versionedTiff.epsg.code, + 'source:epsg': tiffItem.epsg.code, }, - geometry: { type: 'Polygon', coordinates: versionedTiff.bounds.toPolygon() } as GeoJSONPolygon, - bbox: versionedTiff.bounds.toBbox(), + geometry: { type: 'Polygon', coordinates: tiffItem.bounds.toPolygon() } as GeoJSONPolygon, + bbox: tiffItem.bounds.toBbox(), collection: CliId, }; diff --git a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts index b30f059a..66b9540e 100644 --- a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts +++ b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts @@ -1,6 +1,6 @@ import { StacItem } from 'stac-ts'; -import { VersionedTiff } from '../mappers/group-by-map-code.js'; +import { TiffItem } from '../types/tiff-item.js'; import { createBaseStacItem } from './create-base-stac-item.js'; /** @@ -11,20 +11,21 @@ import { createBaseStacItem } from './create-base-stac-item.js'; * All versions need a StacItem object that lives in the topo[50/250] directory * The latest version needs a second StacItem object that lives in the topo[50/250]-latest dir */ -export async function createStacItemGroups( - mapCode: string, - latest: VersionedTiff, - others: VersionedTiff[], +export async function createStacItemPair( scale: string, -): Promise<{ latest: StacItem; all: StacItem[] }> { - const latestStacItem = createBaseStacItem(mapCode, mapCode, latest); - const allStacItems = [...others, latest].map((versionedTiff) => - createBaseStacItem(`${mapCode}_${versionedTiff.version}`, mapCode, versionedTiff), - ); + mapCode: string, + all: TiffItem[], + latest: TiffItem, +): Promise<{ latest: { item: TiffItem; stac: StacItem }; all: { item: TiffItem; stac: StacItem }[] }> { + const latestStacItem = { item: latest, stac: createBaseStacItem(mapCode, mapCode, latest) }; + const allStacItems = all.map((tiffItem) => ({ + item: tiffItem, + stac: createBaseStacItem(`${mapCode}_${tiffItem.version}`, mapCode, tiffItem), + })); // add link to all items pointing to the latest version - allStacItems.forEach((item) => { - item?.links.push({ + allStacItems.forEach((pair) => { + pair.stac?.links.push({ href: `./${mapCode}_${latest.version}.json`, rel: 'latest-version', type: 'application/json', @@ -32,7 +33,7 @@ export async function createStacItemGroups( }); // add link to the latest item referencing its copy that will live in the topo[50/250] directory - latestStacItem.links.push({ + latestStacItem.stac.links.push({ href: `../${scale}/${mapCode}_${latest.version}.json`, rel: 'derived_from', type: 'application/json', diff --git a/src/commands/basemaps-topo-import/stac/write-stac-files.ts b/src/commands/basemaps-topo-import/stac/write-stac-files.ts index b612f69b..96cebd1f 100644 --- a/src/commands/basemaps-topo-import/stac/write-stac-files.ts +++ b/src/commands/basemaps-topo-import/stac/write-stac-files.ts @@ -3,7 +3,6 @@ import { StacCollection, StacItem } from 'stac-ts'; import { logger } from '../../../log.js'; import { isArgo } from '../../../utils/argo.js'; -import { brokenTiffs } from '../topo-stac-creation.js'; export async function writeStacFiles( target: URL, @@ -22,7 +21,4 @@ export async function writeStacFiles( const collectionPath = new URL('collection.json', target); await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); } - - const brokenPath = new URL('/tmp/topo-stac-creation/output/broken.json', target); - await fsa.write(brokenPath, JSON.stringify(Array.from(brokenTiffs.keys()), null, 2)); } diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 209ef67b..cef7131b 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -112,6 +112,7 @@ async function createCogs(input: URL, tmp: URL): Promise { // run gdal commands for each the source file logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); + const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath]); await new GdalRunner(commandBuildVrt).run(logger); @@ -122,7 +123,7 @@ async function createCogs(input: URL, tmp: URL): Promise { const sourceProj = Epsg.tryGet(sourceEpsg); if (sourceProj == null) throw new Error(`Unknown source projection ${sourceEpsg}`); const vrtWarpPath = new URL(`${item.id}-warp.vrt`, tmpFolder); - const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj, sourceRes); + const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj); await new GdalRunner(commandBuildVrtWarp).run(logger); logger.info({ item: item.id }, 'CogCreation:gdal_translate'); diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index bd575fe2..837ee972 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -1,21 +1,23 @@ import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; import { Bounds } from '@basemaps/geo'; import { fsa } from '@basemaps/shared'; -import { Tiff } from '@cogeotiff/core'; import { command, option, string } from 'cmd-ts'; import pLimit from 'p-limit'; import { StacItem } from 'stac-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; +import { isArgo } from '../../utils/argo.js'; import { config, forceOutput, registerCli, tryParseUrl, UrlFolder, verbose } from '../common.js'; -import { groupTiffsByMapCodeAndLatest } from './mappers/group-by-map-code.js'; +import { groupTiffsByDirectory } from './mappers/group-tiffs-by-directory.js'; import { createStacCollection } from './stac/create-stac-collection.js'; -import { createStacItemGroups } from './stac/create-stac-item-groups.js'; +import { createStacItemPair } from './stac/create-stac-item-groups.js'; import { writeStacFiles } from './stac/write-stac-files.js'; +import { ByDirectory } from './types/by-directory.js'; +import { TiffItem } from './types/tiff-item.js'; const Q = pLimit(10); -export const brokenTiffs = new Map(); +export const brokenTiffs = { noBounds: [] as string[], noEpsg: [] as string[] }; /** * List all the tiffs in a directory for topographic maps and create cogs for each. @@ -70,11 +72,18 @@ export const topoStacCreation = command({ const paths: string[] = []; all.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, args.target).href)); - latest.forEach((item) => paths.push(new URL(`${args.scale}-latest/${item.id}.json`, args.target).href)); + latest.forEach((item) => paths.push(new URL(`${args.scale}_latest/${item.id}.json`, args.target).href)); // write stac items into an JSON array - await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/tiles.json`), JSON.stringify(paths, null, 2)); - await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/broken.json`), JSON.stringify(brokenTiffs, null, 2)); + if (args.forceOutput || isArgo()) { + if (isArgo()) { + await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/tiles.json`), JSON.stringify(paths, null, 2)); + await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/brokenTiffs.json`), JSON.stringify(brokenTiffs, null, 2)); + } else { + await fsa.write(new URL('tiles.json', args.target), JSON.stringify(paths, null, 2)); + await fsa.write(new URL('brokenTiffs.json', args.target), JSON.stringify(brokenTiffs, null, 2)); + } + } logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); }, @@ -106,40 +115,81 @@ async function loadTiffsToCreateStacs( const tiffs = await loadTiffsFromPaths(fileURLs, Q); logger.info({ numTiffs: tiffs.length }, 'LoadTiffs:End'); + // group all of the Tiff objects by epsg and map code logger.info('GroupTiffs:Start'); - // group all of the Tiff objects by map code, version, and latest - const groupsByMapCode = await groupTiffsByMapCodeAndLatest(tiffs); - logger.info({ numGroups: groupsByMapCode.size }, 'GroupTiffs:End'); + const itemsByDir = await groupTiffsByDirectory(tiffs); + const itemsByDirPath = new URL('itemsByDirectory.json', target); + await fsa.write(itemsByDirPath, JSON.stringify(itemsByDir, null, 2)); + + logger.info('GroupTiffs:End'); + + // pair all of the TiffItem objects with StacItem objects logger.info('CreateStacItems:Start'); - const latestBounds: Bounds[] = []; - const allBounds: Bounds[] = []; - const latestItems: StacItem[] = []; - const allItems: StacItem[] = []; + const pairsByDir = new ByDirectory<{ item: TiffItem; stac: StacItem }>(); - for (const [mapCode, { latest, others }] of groupsByMapCode.entries()) { - // push bounds to their respective arrays - latestBounds.push(latest.bounds); - allBounds.push(...[latest, ...others].map((vt) => vt.bounds)); + for (const [epsg, itemsByMapCode] of itemsByDir.all.entries()) { + for (const [mapCode, items] of itemsByMapCode.entries()) { + // get latest item + const latest = itemsByDir.latest.get(epsg).get(mapCode); - // create StacItem object groups - const stacItems = await createStacItemGroups(mapCode, latest, others, scale); + // create stac items + const stacItems = await createStacItemPair(scale, mapCode, items, latest); - // push StacItem objects to their respective arrays - latestItems.push(stacItems.latest); - allItems.push(...stacItems.all); + // store stac items + pairsByDir.all.get(epsg).set(mapCode, stacItems.all); + pairsByDir.latest.get(epsg).set(mapCode, stacItems.latest); + } } - logger.info({ numLatestItems: latestItems.length, numItems: allItems.length }, 'CreateStacItems:End'); - // Create collection json for all topo50 items - logger.info('CreateStacCollections:Start'); - const latestCollection = createStacCollection(title, Bounds.union(latestBounds), latestItems); - const nonLatestCollection = createStacCollection(title, Bounds.union(allBounds), allItems); + // const pairsByDirPath = new URL('pairsByDirectory.json', target); + // await fsa.write(pairsByDirPath, JSON.stringify(pairsByDir, null, 2)); + + logger.info('CreateStacItems:End'); + + logger.info('WriteStacFiles:Start'); + + const latestItems: StacItem[] = []; + const allItems: StacItem[] = []; + + // write 'all' stac items and collection + for (const [epsg, pairsByMapCode] of pairsByDir.all.entries()) { + const boundsByEpsg: Bounds[] = []; + const itemsByEpsg: StacItem[] = []; + + for (const [, pairs] of pairsByMapCode.entries()) { + for (const pair of pairs) { + boundsByEpsg.push(pair.item.bounds); + itemsByEpsg.push(pair.stac); + allItems.push(pair.stac); + } + } + + // create collection + const collection = createStacCollection(title, Bounds.union(boundsByEpsg), itemsByEpsg); + // write stac items and collection + await writeStacFiles(new URL(`${scale}/${epsg}/`, target), force, itemsByEpsg, collection); + } + + // write 'latest' stac items and collection + for (const [epsg, itemsByMapCode] of pairsByDir.latest.entries()) { + const boundsByEpsg: Bounds[] = []; + const itemsByEpsg: StacItem[] = []; + + for (const [, pair] of itemsByMapCode.entries()) { + boundsByEpsg.push(pair.item.bounds); + itemsByEpsg.push(pair.stac); + latestItems.push(pair.stac); + } + + // create collection + const collection = createStacCollection(title, Bounds.union(boundsByEpsg), itemsByEpsg); + // write stac items and collection + await writeStacFiles(new URL(`${scale}_latest/${epsg}/`, target), force, itemsByEpsg, collection); + } - await writeStacFiles(new URL(`${scale}-latest/`, target), force, latestItems, latestCollection); - await writeStacFiles(new URL(`${scale}/`, target), force, allItems, nonLatestCollection); - logger.info({ tiffs: tiffs.length }, 'CreateStacCollections:End'); + logger.info('WriteStacFiles:End'); return { latest: latestItems, all: allItems }; } diff --git a/src/commands/basemaps-topo-import/types/by-directory.ts b/src/commands/basemaps-topo-import/types/by-directory.ts new file mode 100644 index 00000000..9b51e7fa --- /dev/null +++ b/src/commands/basemaps-topo-import/types/by-directory.ts @@ -0,0 +1,110 @@ +// ByDirectory { +// all: { +// [epsg: string]: { +// [mapCode: string]: T[] +// } +// }, +// latest: { +// [epsg: string]: { +// [mapCode: string]: T +// } +// } +// } +export class ByDirectory { + readonly all: ByEpsg; + readonly latest: ByEpsg; + + constructor() { + this.all = new ByEpsg(); + this.latest = new ByEpsg(); + } + + toJSON(): object { + return { + all: this.all.toJSON(), + latest: this.latest.toJSON(), + }; + } +} + +// ByEpsg { +// [epsg: string]: { +// [mapCode: string]: T +// } +// } +class ByEpsg { + private readonly items: { [epsg: string]: ByMapCode }; + + constructor() { + this.items = {}; + } + + get(epsg: string): ByMapCode { + let result = this.items[epsg]; + + if (result !== undefined) return result; + + result = new ByMapCode(); + this.items[epsg] = result; + + return result; + } + + /** + * @returns [epsg, ByMapCode][] + */ + entries(): [string, ByMapCode][] { + return Object.entries(this.items); + } + + toJSON(): object { + return Object.entries(this.items).reduce((obj, [epsg, byMapCode]) => ({ ...obj, [epsg]: byMapCode.toJSON() }), {}); + } +} + +// ByMapCode { +// [mapCode: string]: T +// } +class ByMapCode { + private readonly items: { [mapCode: string]: T }; + + constructor() { + this.items = {}; + } + + /** + * @param mapCode: the map code to lookup + * @param defaultValue: the value to set and return if no value is found for the given map code + * + * @returns the value for the given map code, otherwise, the default value + * @throws {Error} if no value is found for the given map code and no default value is provided. + */ + get(mapCode: string, defaultValue?: T): T { + let result = this.items[mapCode]; + + if (result !== undefined) return result; + if (defaultValue === undefined) { + throw new Error(`No value found for map code '${mapCode}'`); + } + + result = defaultValue; + this.items[mapCode] = result; + + return result; + } + + set(mapCode: string, value: T): void { + this.items[mapCode] = value; + } + + /** + * @returns [mapCode, T][] + */ + entries(): [string, T][] { + return Object.entries(this.items); + } + + toJSON(): object { + return Object.entries(this.items).reduce((obj, [mapCode, value]) => ({ ...obj, [mapCode]: value }), {}); + } +} diff --git a/src/commands/basemaps-topo-import/types/tiff-item.ts b/src/commands/basemaps-topo-import/types/tiff-item.ts new file mode 100644 index 00000000..7cde6d65 --- /dev/null +++ b/src/commands/basemaps-topo-import/types/tiff-item.ts @@ -0,0 +1,24 @@ +import { Bounds, Epsg } from '@basemaps/geo'; +import { Tiff } from '@basemaps/shared'; + +export class TiffItem { + readonly tiff: Tiff; + readonly source: URL; + readonly mapCode: string; + readonly version: string; + readonly bounds: Bounds; + readonly epsg: Epsg; + + constructor(tiff: Tiff, source: URL, mapCode: string, version: string, bounds: Bounds, epsg: Epsg) { + this.tiff = tiff; + this.source = source; + this.mapCode = mapCode; + this.version = version; + this.bounds = bounds; + this.epsg = epsg; + } + + toJSON(): string { + return `${this.mapCode}_${this.version}`; + } +} From 39947382d2bfa17738d6b57e62d97c4901ab884f Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Mon, 9 Dec 2024 15:17:12 +1300 Subject: [PATCH 29/47] fix: updated processes to passthrough tiff metadata --- .../basemaps-topo-import/gdal-commands.ts | 8 +- .../mappers/group-tiffs-by-directory.ts | 13 +- .../stac/create-base-stac-item.ts | 29 ++-- .../stac/create-stac-collection.ts | 7 +- .../stac/create-stac-item-groups.ts | 27 ++-- .../stac/write-stac-files.ts | 28 ++-- .../basemaps-topo-import/topo-cog-creation.ts | 2 +- .../topo-stac-creation.ts | 132 ++++++++---------- 8 files changed, 113 insertions(+), 133 deletions(-) diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts index 70e225ad..4d15691c 100644 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ b/src/commands/basemaps-topo-import/gdal-commands.ts @@ -8,7 +8,13 @@ export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { return { output: targetVrt, command: 'gdalbuildvrt', - args: [['-addalpha'], urlToString(targetVrt), ...source.map(urlToString)] + args: [ + ['-addalpha'], + ['-hidenodata'], + ['-vrtnodata', '208 231 244'], + urlToString(targetVrt), + ...source.map(urlToString), + ] .filter((f) => f != null) .flat() .map(String), diff --git a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts index c5ded1b5..bf560a07 100644 --- a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts +++ b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts @@ -1,4 +1,3 @@ -import { Bounds, Projection } from '@basemaps/geo'; import { Tiff } from '@cogeotiff/core'; import { logger } from '../../../log.js'; @@ -14,8 +13,8 @@ import { TiffItem } from '../types/tiff-item.js'; * For each group, we then need to identify the latest version and set it aside from the rest. * The latest version will have special metadata, whereas the rest will have similar metadata. * - * @param tiffs: The tiffs to group by map code and version - * @returns a `SplitEpsgMap` Map object + * @param tiffs: The tiffs to group by epsg, and map code + * @returns a `ByDirectory` promise */ export async function groupTiffsByDirectory(tiffs: Tiff[]): Promise> { // group the tiffs by directory, epsg, and map code @@ -43,13 +42,7 @@ export async function groupTiffsByDirectory(tiffs: Tiff[]): Promise { - const latestStacItem = { item: latest, stac: createBaseStacItem(mapCode, mapCode, latest) }; - const allStacItems = all.map((tiffItem) => ({ - item: tiffItem, - stac: createBaseStacItem(`${mapCode}_${tiffItem.version}`, mapCode, tiffItem), - })); +): Promise<{ all: StacItem[]; latest: StacItem }> { + const allStacItems = all.map((item) => createBaseStacItem(`${item.mapCode}_${item.version}`, item)); + + const latestURL = new URL(`${latest.mapCode}_${latest.version}.json`, allTargetURL); // add link to all items pointing to the latest version - allStacItems.forEach((pair) => { - pair.stac?.links.push({ - href: `./${mapCode}_${latest.version}.json`, + allStacItems.forEach((stacItem) => { + stacItem.links.push({ + href: latestURL.href, rel: 'latest-version', type: 'application/json', }); }); + const latestStacItem = createBaseStacItem(latest.mapCode, latest); + // add link to the latest item referencing its copy that will live in the topo[50/250] directory - latestStacItem.stac.links.push({ - href: `../${scale}/${mapCode}_${latest.version}.json`, + latestStacItem.links.push({ + href: latestURL.href, rel: 'derived_from', type: 'application/json', }); diff --git a/src/commands/basemaps-topo-import/stac/write-stac-files.ts b/src/commands/basemaps-topo-import/stac/write-stac-files.ts index 96cebd1f..c0942b0f 100644 --- a/src/commands/basemaps-topo-import/stac/write-stac-files.ts +++ b/src/commands/basemaps-topo-import/stac/write-stac-files.ts @@ -2,23 +2,27 @@ import { fsa } from '@basemaps/shared'; import { StacCollection, StacItem } from 'stac-ts'; import { logger } from '../../../log.js'; -import { isArgo } from '../../../utils/argo.js'; export async function writeStacFiles( target: URL, - force: boolean, items: StacItem[], collection: StacCollection, -): Promise { +): Promise<{ itemPaths: URL[]; collectionPath: URL }> { // Create collection json for all topo50-latest items. - if (force || isArgo()) { - logger.info({ target }, 'CreateStac:Output'); - logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); - for (const item of items) { - const itemPath = new URL(`${item.id}.json`, target); - await fsa.write(itemPath, JSON.stringify(item, null, 2)); - } - const collectionPath = new URL('collection.json', target); - await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); + logger.info({ target }, 'CreateStac:Output'); + logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); + + const itemPaths: URL[] = []; + + for (const item of items) { + const itemPath = new URL(`${item.id}.json`, target); + itemPaths.push(itemPath); + + await fsa.write(itemPath, JSON.stringify(item, null, 2)); } + + const collectionPath = new URL('collection.json', target); + await fsa.write(collectionPath, JSON.stringify(collection, null, 2)); + + return { itemPaths, collectionPath }; } diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index cef7131b..1ab2c1bf 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -119,7 +119,7 @@ async function createCogs(input: URL, tmp: URL): Promise { logger.info({ item: item.id }, 'CogCreation:gdalwarp'); const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const sourceEpsg = Number(item.properties['source:epsg']); + const sourceEpsg = Number(item.properties['proj:epsg']); const sourceProj = Epsg.tryGet(sourceEpsg); if (sourceProj == null) throw new Error(`Unknown source projection ${sourceEpsg}`); const vrtWarpPath = new URL(`${item.id}-warp.vrt`, tmpFolder); diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 837ee972..c9ed4ce1 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -1,7 +1,7 @@ import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; import { Bounds } from '@basemaps/geo'; import { fsa } from '@basemaps/shared'; -import { command, option, string } from 'cmd-ts'; +import { boolean, command, flag, option, string } from 'cmd-ts'; import pLimit from 'p-limit'; import { StacItem } from 'stac-ts'; @@ -11,10 +11,8 @@ import { isArgo } from '../../utils/argo.js'; import { config, forceOutput, registerCli, tryParseUrl, UrlFolder, verbose } from '../common.js'; import { groupTiffsByDirectory } from './mappers/group-tiffs-by-directory.js'; import { createStacCollection } from './stac/create-stac-collection.js'; -import { createStacItemPair } from './stac/create-stac-item-groups.js'; +import { createStacItems } from './stac/create-stac-item-groups.js'; import { writeStacFiles } from './stac/write-stac-files.js'; -import { ByDirectory } from './types/by-directory.js'; -import { TiffItem } from './types/tiff-item.js'; const Q = pLimit(10); export const brokenTiffs = { noBounds: [] as string[], noEpsg: [] as string[] }; @@ -53,7 +51,14 @@ export const topoStacCreation = command({ scale: option({ type: string, long: 'scale', - description: 'topo50 or topo250', + description: 'topo25, topo50, or topo250', + }), + latestOnly: flag({ + type: boolean, + defaultValue: () => false, + long: 'latest-only', + description: 'Only process the latest version of each map sheet', + defaultValueIsSerializable: true, }), }, async handler(args) { @@ -61,28 +66,28 @@ export const topoStacCreation = command({ registerCli(this, args); logger.info('ListJobs:Start'); - const { latest, all } = await loadTiffsToCreateStacs( + const { epsgDirectoryPaths, stacItemPaths } = await loadTiffsToCreateStacs( + args.latestOnly, args.source, args.target, args.title, args.forceOutput, args.scale, ); - if (latest.length === 0 || all.length === 0) throw new Error('No Stac items created'); - const paths: string[] = []; - all.forEach((item) => paths.push(new URL(`${args.scale}/${item.id}.json`, args.target).href)); - latest.forEach((item) => paths.push(new URL(`${args.scale}_latest/${item.id}.json`, args.target).href)); + if (epsgDirectoryPaths.length === 0 || stacItemPaths.length === 0) throw new Error('No Stac items created'); // write stac items into an JSON array if (args.forceOutput || isArgo()) { - if (isArgo()) { - await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/tiles.json`), JSON.stringify(paths, null, 2)); - await fsa.write(tryParseUrl(`/tmp/topo-stac-creation/brokenTiffs.json`), JSON.stringify(brokenTiffs, null, 2)); - } else { - await fsa.write(new URL('tiles.json', args.target), JSON.stringify(paths, null, 2)); - await fsa.write(new URL('brokenTiffs.json', args.target), JSON.stringify(brokenTiffs, null, 2)); - } + const targetURL = isArgo() ? tryParseUrl('/tmp/topo-stac-creation/') : args.target; + + // for create-config: we need to tell create-config to create a bundled config for each epsg folder (latest only). + // workflow: will loop 'targets.json' and create a node for each path where each node's job is to create a bundled config. + await fsa.write(new URL('targets.json', targetURL), JSON.stringify(epsgDirectoryPaths, null, 2)); + + // tiles.json makes the tiff files + await fsa.write(new URL('tiles.json', targetURL), JSON.stringify(stacItemPaths, null, 2)); + await fsa.write(new URL('brokenTiffs.json', targetURL), JSON.stringify(brokenTiffs, null, 2)); } logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); @@ -102,12 +107,13 @@ export const topoStacCreation = command({ * @returns an array of StacItem objects */ async function loadTiffsToCreateStacs( + latestOnly: boolean, source: URL, target: URL, title: string, force: boolean, scale: string, -): Promise<{ latest: StacItem[]; all: StacItem[] }> { +): Promise<{ epsgDirectoryPaths: URL[]; stacItemPaths: URL[] }> { logger.info({ source }, 'LoadTiffs:Start'); // extract all file paths from the source directory and convert them into URL objects const fileURLs = await fsa.toArray(fsa.list(source)); @@ -117,79 +123,61 @@ async function loadTiffsToCreateStacs( // group all of the Tiff objects by epsg and map code logger.info('GroupTiffs:Start'); - const itemsByDir = await groupTiffsByDirectory(tiffs); const itemsByDirPath = new URL('itemsByDirectory.json', target); await fsa.write(itemsByDirPath, JSON.stringify(itemsByDir, null, 2)); - logger.info('GroupTiffs:End'); - // pair all of the TiffItem objects with StacItem objects - logger.info('CreateStacItems:Start'); - - const pairsByDir = new ByDirectory<{ item: TiffItem; stac: StacItem }>(); + const epsgDirectoryPaths: URL[] = []; + const stacItemPaths: URL[] = []; + // create and write stac items and collections for (const [epsg, itemsByMapCode] of itemsByDir.all.entries()) { + const allTargetURL = new URL(`${scale}/${epsg}/`, target); + const latestTargetURL = new URL(`${scale}_latest/${epsg}/`, target); + + const allBounds: Bounds[] = []; + const allStacItems: StacItem[] = []; + + const latestBounds: Bounds[] = []; + const latestStacItems: StacItem[] = []; + + // create stac items + logger.info({ epsg }, 'CreateStacItems:Start'); for (const [mapCode, items] of itemsByMapCode.entries()) { // get latest item const latest = itemsByDir.latest.get(epsg).get(mapCode); // create stac items - const stacItems = await createStacItemPair(scale, mapCode, items, latest); - - // store stac items - pairsByDir.all.get(epsg).set(mapCode, stacItems.all); - pairsByDir.latest.get(epsg).set(mapCode, stacItems.latest); - } - } - - // const pairsByDirPath = new URL('pairsByDirectory.json', target); - // await fsa.write(pairsByDirPath, JSON.stringify(pairsByDir, null, 2)); - - logger.info('CreateStacItems:End'); - - logger.info('WriteStacFiles:Start'); - - const latestItems: StacItem[] = []; - const allItems: StacItem[] = []; + const stacItems = await createStacItems(allTargetURL, items, latest); - // write 'all' stac items and collection - for (const [epsg, pairsByMapCode] of pairsByDir.all.entries()) { - const boundsByEpsg: Bounds[] = []; - const itemsByEpsg: StacItem[] = []; + allBounds.push(...items.map((item) => item.bounds)); + allStacItems.push(...stacItems.all); - for (const [, pairs] of pairsByMapCode.entries()) { - for (const pair of pairs) { - boundsByEpsg.push(pair.item.bounds); - itemsByEpsg.push(pair.stac); - allItems.push(pair.stac); - } + latestBounds.push(latest.bounds); + latestStacItems.push(stacItems.latest); } - // create collection - const collection = createStacCollection(title, Bounds.union(boundsByEpsg), itemsByEpsg); - // write stac items and collection - await writeStacFiles(new URL(`${scale}/${epsg}/`, target), force, itemsByEpsg, collection); - } + // create collections + const collection = createStacCollection(title, Bounds.union(allBounds), allStacItems); + const latestCollection = createStacCollection(title, Bounds.union(latestBounds), latestStacItems); + logger.info({ epsg }, 'CreateStacItems:End'); - // write 'latest' stac items and collection - for (const [epsg, itemsByMapCode] of pairsByDir.latest.entries()) { - const boundsByEpsg: Bounds[] = []; - const itemsByEpsg: StacItem[] = []; + if (force || isArgo()) { + // epsgDirectoryPaths.push(allTargetURL); + epsgDirectoryPaths.push(latestTargetURL); - for (const [, pair] of itemsByMapCode.entries()) { - boundsByEpsg.push(pair.item.bounds); - itemsByEpsg.push(pair.stac); - latestItems.push(pair.stac); + // write stac items and collections + logger.info({ epsg }, 'WriteStacFiles:Start'); + if (!latestOnly) { + const allPaths = await writeStacFiles(allTargetURL, allStacItems, collection); + stacItemPaths.push(...allPaths.itemPaths); + } + const latestPaths = await writeStacFiles(latestTargetURL, latestStacItems, latestCollection); + stacItemPaths.push(...latestPaths.itemPaths); + logger.info({ epsg }, 'WriteStacFiles:End'); } - - // create collection - const collection = createStacCollection(title, Bounds.union(boundsByEpsg), itemsByEpsg); - // write stac items and collection - await writeStacFiles(new URL(`${scale}_latest/${epsg}/`, target), force, itemsByEpsg, collection); } - logger.info('WriteStacFiles:End'); - - return { latest: latestItems, all: allItems }; + return { epsgDirectoryPaths, stacItemPaths }; } From 3800b9d493f72cde1733f980dd72e6efa4a821b2 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Mon, 9 Dec 2024 16:05:56 +1300 Subject: [PATCH 30/47] fix: update target.json artifact format to work with Argo 'withParam' --- src/commands/basemaps-topo-import/topo-stac-creation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index c9ed4ce1..55ceb955 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -83,7 +83,8 @@ export const topoStacCreation = command({ // for create-config: we need to tell create-config to create a bundled config for each epsg folder (latest only). // workflow: will loop 'targets.json' and create a node for each path where each node's job is to create a bundled config. - await fsa.write(new URL('targets.json', targetURL), JSON.stringify(epsgDirectoryPaths, null, 2)); + const targetPaths = epsgDirectoryPaths.map((url) => ({ url })); + await fsa.write(new URL('targets.json', targetURL), JSON.stringify(targetPaths, null, 2)); // tiles.json makes the tiff files await fsa.write(new URL('tiles.json', targetURL), JSON.stringify(stacItemPaths, null, 2)); From 6ef612fe851dcc1d4d4bf4a7c024a3047e3a2f8a Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Tue, 10 Dec 2024 09:37:10 +1300 Subject: [PATCH 31/47] refactor: split the gdal processes into functions and added option support --- .../basemaps-topo-import/gdal-commands.ts | 75 ------------------- .../gdal/gdal-build-cog.ts | 60 +++++++++++++++ .../gdal/gdal-build-vrt-warp.ts | 55 ++++++++++++++ .../gdal/gdal-build-vrt.ts | 55 ++++++++++++++ .../basemaps-topo-import/gdal/gdal-opts.ts | 6 ++ .../basemaps-topo-import/topo-cog-creation.ts | 57 +++++++++++--- 6 files changed, 223 insertions(+), 85 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/gdal-commands.ts create mode 100644 src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts create mode 100644 src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts create mode 100644 src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts create mode 100644 src/commands/basemaps-topo-import/gdal/gdal-opts.ts diff --git a/src/commands/basemaps-topo-import/gdal-commands.ts b/src/commands/basemaps-topo-import/gdal-commands.ts deleted file mode 100644 index 4d15691c..00000000 --- a/src/commands/basemaps-topo-import/gdal-commands.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { Epsg } from '@basemaps/geo'; - -import { urlToString } from '../common.js'; - -export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { - if (source.length === 0) throw new Error('No source files given for :' + targetVrt.href); - return { - output: targetVrt, - command: 'gdalbuildvrt', - args: [ - ['-addalpha'], - ['-hidenodata'], - ['-vrtnodata', '208 231 244'], - urlToString(targetVrt), - ...source.map(urlToString), - ] - .filter((f) => f != null) - .flat() - .map(String), - }; -} - -export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { - return { - output: targetVrt, - command: 'gdalwarp', - args: [ - ['-multi'], // Mutithread IO - ['-of', 'vrt'], // Output as a VRT - ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp - ['-s_srs', sourceProj.toEpsgString()], // Source EPSG - // ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG - // ['-tr', targetResolution, targetResolution], - urlToString(sourceVrt), - urlToString(targetVrt), - ] - .filter((f) => f != null) - .flat() - .map(String), - }; -} - -export function gdalBuildCogCommands(input: URL, output: URL): GdalCommand { - return { - output, - command: 'gdal_translate', - args: [ - ['-q'], // Supress non-error output - ['-stats'], // Force stats (re)computation - ['-of', 'COG'], // Output format - - // https://gdal.org/en/latest/drivers/raster/cog.html#creation-options - ['-co', 'BIGTIFF=NO'], - ['-co', 'BLOCKSIZE=512'], - ['-co', 'COMPRESS=WEBP'], - ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS - ['-co', 'OVERVIEW_COMPRESS=WEBP'], - ['-co', 'OVERVIEWS=IGNORE_EXISTING'], - ['-co', 'OVERVIEW_QUALITY=90'], - ['-co', 'OVERVIEW_RESAMPLING=LANCZOS'], - ['-co', 'QUALITY=100'], - ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes - - // https://gdal.org/en/latest/drivers/raster/cog.html#reprojection-related-creation-options - ['-co', 'ADD_ALPHA=YES'], - - urlToString(input), - urlToString(output), - ] - .filter((f) => f != null) - .flat() - .map(String), - }; -} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts new file mode 100644 index 00000000..5b295b35 --- /dev/null +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts @@ -0,0 +1,60 @@ +import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; + +import { urlToString } from '../../common.js'; + +interface gdalBuildCogOptions { + /** + * Resampling algorithm. Nearest is the default. + * + * @link + * https://gdal.org/en/stable/programs/gdal_translate.html#cmdoption-gdal_translate-r + */ + resamplingMethod?: GdalResampling; +} + +/** + * Constructs a 'gdal_translate' GdalCommand. + * + * @param input + * @param output + * @param opts + * @returns + */ +export function gdalBuildCogCommands(input: URL, output: URL, opts?: gdalBuildCogOptions): GdalCommand { + const command: GdalCommand = { + output, + command: 'gdal_translate', + args: [ + ['-q'], // Supress non-error output + ['-stats'], // Force stats (re)computation + ['-of', 'COG'], // Output format + + // https://gdal.org/en/latest/drivers/raster/cog.html#creation-options + ['-co', 'BIGTIFF=NO'], + ['-co', 'BLOCKSIZE=512'], + ['-co', 'COMPRESS=WEBP'], + ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS + ['-co', 'OVERVIEW_COMPRESS=WEBP'], + ['-co', 'OVERVIEWS=IGNORE_EXISTING'], + ['-co', 'OVERVIEW_QUALITY=90'], + ['-co', 'OVERVIEW_RESAMPLING=LANCZOS'], + ['-co', 'QUALITY=100'], + ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes + + // https://gdal.org/en/latest/drivers/raster/cog.html#reprojection-related-creation-options + ['-co', 'ADD_ALPHA=YES'], + ] + .filter((f) => f != null) + .flat() + .map(String), + }; + + if (opts?.resamplingMethod != null) { + command.args.push('-r', opts.resamplingMethod); + } + + command.args.push(urlToString(input), urlToString(output)); + + return command; +} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts new file mode 100644 index 00000000..12f7b49b --- /dev/null +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts @@ -0,0 +1,55 @@ +import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; +import { Epsg } from '@basemaps/geo'; + +import { urlToString } from '../../common.js'; + +interface gdalBuildVrtWarpOptions { + /** + * Resampling algorithm. Nearest is the default. + * + * @link + * https://gdal.org/en/latest/programs/gdalwarp.html#cmdoption-gdalwarp-r + */ + resamplingMethod?: GdalResampling; +} + +/** + * Constructs a 'gdalwarp' GdalCommand. + * + * @param targetVrt + * @param sourceVrt + * @param sourceProj + * @param opts + * @returns + */ +export function gdalBuildVrtWarp( + targetVrt: URL, + sourceVrt: URL, + sourceProj: Epsg, + opts?: gdalBuildVrtWarpOptions, +): GdalCommand { + const command: GdalCommand = { + output: targetVrt, + command: 'gdalwarp', + args: [ + ['-multi'], // Mutithread IO + ['-of', 'vrt'], // Output as a VRT + ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp + ['-s_srs', sourceProj.toEpsgString()], // Source EPSG + // ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG + // ['-tr', targetResolution, targetResolution], + ] + .filter((f) => f != null) + .flat() + .map(String), + }; + + if (opts?.resamplingMethod != null) { + command.args.push('-r', opts.resamplingMethod); + } + + command.args.push(urlToString(sourceVrt), urlToString(targetVrt)); + + return command; +} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts new file mode 100644 index 00000000..55c195a8 --- /dev/null +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts @@ -0,0 +1,55 @@ +import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; +import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; + +import { urlToString } from '../../common.js'; + +interface GdalBuildVrtOptions { + /** + * Resampling algorithm. Nearest is the default. + * + * @link + * https://gdal.org/en/latest/programs/gdalbuildvrt.html#cmdoption-gdalbuildvrt-r + */ + resamplingMethod?: GdalResampling; + + /** + * Background color. + * + * @example + * '208 231 244' // NZ Topo Raster sea colour (RGB as 'RRR GGG BBB') + * + * @links + * https://gdal.org/en/latest/programs/gdalbuildvrt.html#cmdoption-gdalbuildvrt-vrtnodata + * https://gdal.org/en/latest/programs/gdalbuildvrt.html#vrtnodata + */ + background?: string; +} + +/** + * Constructs a 'gdalBuildVrt' GdalCommand. + * + * @param targetVrt + * @param source + * @param opts + * @returns + */ +export function gdalBuildVrt(targetVrt: URL, source: URL[], opts?: GdalBuildVrtOptions): GdalCommand { + if (source.length === 0) throw new Error('No source files given for :' + targetVrt.href); + const command: GdalCommand = { + output: targetVrt, + command: 'gdalbuildvrt', + args: ['-addalpha'], + }; + + if (opts?.background != null) { + command.args.push('-hidenodata', '-vrtnodata', opts.background); + } + + if (opts?.resamplingMethod != null) { + command.args.push('-r', opts.resamplingMethod); + } + + command.args.push(urlToString(targetVrt), ...source.map(urlToString)); + + return command; +} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-opts.ts b/src/commands/basemaps-topo-import/gdal/gdal-opts.ts new file mode 100644 index 00000000..13b105f6 --- /dev/null +++ b/src/commands/basemaps-topo-import/gdal/gdal-opts.ts @@ -0,0 +1,6 @@ +const resamplingOptions = ['nearest', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'average', 'mode'] as const; +type ResamplingOption = (typeof resamplingOptions)[number]; + +export function checkResamplingMethod(resamplingMethod: string): resamplingMethod is ResamplingOption { + return (resamplingOptions as readonly string[]).includes(resamplingMethod); +} diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 1ab2c1bf..af58181e 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -1,7 +1,7 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; +import { createFileStats, GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; import { Epsg } from '@basemaps/geo'; import { fsa, Tiff } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; @@ -16,7 +16,11 @@ import { logger } from '../../log.js'; import { HashTransform } from '../../utils/hash.stream.js'; import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; import { loadInput } from '../group/group.js'; -import { gdalBuildCogCommands, gdalBuildVrt, gdalBuildVrtWarp } from './gdal-commands.js'; +import { gdalBuildCogCommands as gdalBuildCog } from './gdal/gdal-build-cog.js'; +import { gdalBuildVrt } from './gdal/gdal-build-vrt.js'; +import { gdalBuildVrtWarp } from './gdal/gdal-build-vrt-warp.js'; +import { checkResamplingMethod as isValidResamplingOption } from './gdal/gdal-opts.js'; + const Q = pLimit(10); /** @@ -45,11 +49,26 @@ export const topoCogCreation = command({ long: 'from-file', description: 'JSON file to load inputs from, must be a JSON Array', }), + resamplingMethod: option({ + type: optional(string), + long: 'resampling-method', + description: 'the GDAL resampling algorithm to use', + }), }, async handler(args) { const startTime = performance.now(); registerCli(this, args); logger.info('ListJobs:Start'); + + // check if resampling method is a valid option + const resamplingMethod = args.resamplingMethod; + + if (resamplingMethod != null) { + if (!isValidResamplingOption(resamplingMethod)) { + throw new Error('the provided resampling method is not a valid option'); + } + } + // Load the items for processing const inputs: string[] = []; for (const input of args.inputs) inputs.push(...loadInput(input)); @@ -70,7 +89,7 @@ export const topoCogCreation = command({ await Promise.all( inputs.map((input) => Q(async () => { - await createCogs(tryParseUrl(input), tmpFolder); + await createCogs(tryParseUrl(input), tmpFolder, resamplingMethod); }), ), ); @@ -79,7 +98,7 @@ export const topoCogCreation = command({ }, }); -async function createCogs(input: URL, tmp: URL): Promise { +async function createCogs(input: URL, tmp: URL, resamplingMethod?: GdalResampling): Promise { const startTime = performance.now(); const item = await fsa.readJson(input); const tmpFolder = new URL(item.id, tmp); @@ -110,24 +129,42 @@ async function createCogs(input: URL, tmp: URL): Promise { const sourceRes = tiff.images[0]?.resolution; if (sourceRes == null) throw new Error('Could not read resolution from first image'); - // run gdal commands for each the source file + // run gdal commands for each source file + + /** + * command: gdal build vrt + */ logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); - const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath]); + const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath], { resamplingMethod }); + await new GdalRunner(commandBuildVrt).run(logger); + /** + * command: gdal build warp vrt + */ logger.info({ item: item.id }, 'CogCreation:gdalwarp'); - const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const sourceEpsg = Number(item.properties['proj:epsg']); + + const sourceEpsg = item.properties['proj:epsg']; + if (typeof sourceEpsg !== 'number') throw new Error(`Could not read 'proj:epsg' property from StacItem`); + const sourceProj = Epsg.tryGet(sourceEpsg); if (sourceProj == null) throw new Error(`Unknown source projection ${sourceEpsg}`); + const vrtWarpPath = new URL(`${item.id}-warp.vrt`, tmpFolder); - const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj); + const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj, { resamplingMethod }); + await new GdalRunner(commandBuildVrtWarp).run(logger); + /** + * command: gdal build cog + */ logger.info({ item: item.id }, 'CogCreation:gdal_translate'); - const commandTranslate = gdalBuildCogCommands(vrtWarpPath, tempPath); + + const tempPath = new URL(`${item.id}.tiff`, tmpFolder); + const commandTranslate = gdalBuildCog(vrtWarpPath, tempPath, { resamplingMethod }); + await new GdalRunner(commandTranslate).run(logger); // fsa.write output to target location From f62220e533938e88eb5fd209768f3e586d3e7a14 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Tue, 10 Dec 2024 11:24:46 +1300 Subject: [PATCH 32/47] fix: updated targets.json to include epsg code with url --- .../topo-stac-creation.ts | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 55ceb955..9fbd3df8 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -53,6 +53,11 @@ export const topoStacCreation = command({ long: 'scale', description: 'topo25, topo50, or topo250', }), + resolution: option({ + type: string, + long: 'resolution', + description: 'e.g. gridless_600dpi', + }), latestOnly: flag({ type: boolean, defaultValue: () => false, @@ -73,6 +78,7 @@ export const topoStacCreation = command({ args.title, args.forceOutput, args.scale, + args.resolution, ); if (epsgDirectoryPaths.length === 0 || stacItemPaths.length === 0) throw new Error('No Stac items created'); @@ -83,8 +89,7 @@ export const topoStacCreation = command({ // for create-config: we need to tell create-config to create a bundled config for each epsg folder (latest only). // workflow: will loop 'targets.json' and create a node for each path where each node's job is to create a bundled config. - const targetPaths = epsgDirectoryPaths.map((url) => ({ url })); - await fsa.write(new URL('targets.json', targetURL), JSON.stringify(targetPaths, null, 2)); + await fsa.write(new URL('targets.json', targetURL), JSON.stringify(epsgDirectoryPaths, null, 2)); // tiles.json makes the tiff files await fsa.write(new URL('tiles.json', targetURL), JSON.stringify(stacItemPaths, null, 2)); @@ -114,7 +119,8 @@ async function loadTiffsToCreateStacs( title: string, force: boolean, scale: string, -): Promise<{ epsgDirectoryPaths: URL[]; stacItemPaths: URL[] }> { + resolution: string, +): Promise<{ epsgDirectoryPaths: { epsg: string; url: URL }[]; stacItemPaths: URL[] }> { logger.info({ source }, 'LoadTiffs:Start'); // extract all file paths from the source directory and convert them into URL objects const fileURLs = await fsa.toArray(fsa.list(source)); @@ -129,13 +135,13 @@ async function loadTiffsToCreateStacs( await fsa.write(itemsByDirPath, JSON.stringify(itemsByDir, null, 2)); logger.info('GroupTiffs:End'); - const epsgDirectoryPaths: URL[] = []; + const epsgDirectoryPaths: { epsg: string; url: URL }[] = []; const stacItemPaths: URL[] = []; // create and write stac items and collections for (const [epsg, itemsByMapCode] of itemsByDir.all.entries()) { - const allTargetURL = new URL(`${scale}/${epsg}/`, target); - const latestTargetURL = new URL(`${scale}_latest/${epsg}/`, target); + const allTargetURL = new URL(`${scale}/${resolution}/${epsg}/`, target); + const latestTargetURL = new URL(`${scale}_latest/${resolution}/${epsg}/`, target); const allBounds: Bounds[] = []; const allStacItems: StacItem[] = []; @@ -165,8 +171,7 @@ async function loadTiffsToCreateStacs( logger.info({ epsg }, 'CreateStacItems:End'); if (force || isArgo()) { - // epsgDirectoryPaths.push(allTargetURL); - epsgDirectoryPaths.push(latestTargetURL); + epsgDirectoryPaths.push({ epsg, url: latestTargetURL }); // write stac items and collections logger.info({ epsg }, 'WriteStacFiles:Start'); From e14d1af537cbad391c4dd82f092b26873ac1fcbd Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 11 Dec 2024 11:42:36 +1300 Subject: [PATCH 33/47] Add the trim pixels --- .../extractors/extract-bounds.ts | 15 ++++++++++++++- .../basemaps-topo-import/gdal/gdal-build-cog.ts | 13 ++++++++++++- .../mappers/group-tiffs-by-directory.ts | 12 +++++++++--- .../stac/create-base-stac-item.ts | 2 ++ .../basemaps-topo-import/topo-cog-creation.ts | 15 +++++++++++---- .../basemaps-topo-import/topo-stac-creation.ts | 2 +- .../basemaps-topo-import/types/tiff-item.ts | 6 ++++-- 7 files changed, 53 insertions(+), 12 deletions(-) diff --git a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts index 9ab361a8..103b699f 100644 --- a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts +++ b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts @@ -1,4 +1,4 @@ -import { Bounds } from '@basemaps/geo'; +import { Bounds, Size } from '@basemaps/geo'; import { Tiff } from '@cogeotiff/core'; import { logger } from '../../../log.js'; @@ -22,3 +22,16 @@ export async function extractBounds(tiff: Tiff): Promise { return null; } } + +/** + * This function attempts to extract bounds from the given Tiff object. + * + * @param tiff: The Tiff object from which to extract bounds + * + * @returns if succeeded, a Bounds object. Otherwise, null. + */ +export function extractSize(tiff: Tiff): Size | null { + const size = tiff.images[0]?.size; + if (size == null || size.width == null || size.height == null) return null; + return size; +} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts index 5b295b35..44e9b567 100644 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts @@ -3,6 +3,8 @@ import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; import { urlToString } from '../../common.js'; +export const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; + interface gdalBuildCogOptions { /** * Resampling algorithm. Nearest is the default. @@ -11,6 +13,7 @@ interface gdalBuildCogOptions { * https://gdal.org/en/stable/programs/gdal_translate.html#cmdoption-gdal_translate-r */ resamplingMethod?: GdalResampling; + pixelTrim?: number; } /** @@ -21,7 +24,14 @@ interface gdalBuildCogOptions { * @param opts * @returns */ -export function gdalBuildCogCommands(input: URL, output: URL, opts?: gdalBuildCogOptions): GdalCommand { +export function gdalBuildCogCommands( + input: URL, + output: URL, + width: number, + height: number, + opts?: gdalBuildCogOptions, +): GdalCommand { + const pixelTrim = opts?.pixelTrim ?? DEFAULT_TRIM_PIXEL_RIGHT; const command: GdalCommand = { output, command: 'gdal_translate', @@ -31,6 +41,7 @@ export function gdalBuildCogCommands(input: URL, output: URL, opts?: gdalBuildCo ['-of', 'COG'], // Output format // https://gdal.org/en/latest/drivers/raster/cog.html#creation-options + ['-srcwin', `0`, `0`, `${width - pixelTrim}`, `${height}`], ['-co', 'BIGTIFF=NO'], ['-co', 'BLOCKSIZE=512'], ['-co', 'COMPRESS=WEBP'], diff --git a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts index bf560a07..a375a87a 100644 --- a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts +++ b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts @@ -1,7 +1,7 @@ import { Tiff } from '@cogeotiff/core'; import { logger } from '../../../log.js'; -import { extractBounds } from '../extractors/extract-bounds.js'; +import { extractBounds, extractSize } from '../extractors/extract-bounds.js'; import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; import { brokenTiffs } from '../topo-stac-creation.js'; @@ -27,8 +27,9 @@ export async function groupTiffsByDirectory(tiffs: Tiff[]): Promise Q(async () => { - await createCogs(tryParseUrl(input), tmpFolder, resamplingMethod); + await createCogs(tryParseUrl(input), tmpFolder, args.pixelTrim, resamplingMethod); }), ), ); @@ -98,7 +103,7 @@ export const topoCogCreation = command({ }, }); -async function createCogs(input: URL, tmp: URL, resamplingMethod?: GdalResampling): Promise { +async function createCogs(input: URL, tmp: URL, pixelTrim?: number, resamplingMethod?: GdalResampling): Promise { const startTime = performance.now(); const item = await fsa.readJson(input); const tmpFolder = new URL(item.id, tmp); @@ -162,8 +167,10 @@ async function createCogs(input: URL, tmp: URL, resamplingMethod?: GdalResamplin */ logger.info({ item: item.id }, 'CogCreation:gdal_translate'); + const width = Number(item.properties['source.width']); + const height = Number(item.properties['source.height']); const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const commandTranslate = gdalBuildCog(vrtWarpPath, tempPath, { resamplingMethod }); + const commandTranslate = gdalBuildCog(vrtWarpPath, tempPath, width, height, { resamplingMethod, pixelTrim }); await new GdalRunner(commandTranslate).run(logger); diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 9fbd3df8..71d43532 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -15,7 +15,7 @@ import { createStacItems } from './stac/create-stac-item-groups.js'; import { writeStacFiles } from './stac/write-stac-files.js'; const Q = pLimit(10); -export const brokenTiffs = { noBounds: [] as string[], noEpsg: [] as string[] }; +export const brokenTiffs = { noBounds: [] as string[], noEpsg: [] as string[], noSize: [] as string[] }; /** * List all the tiffs in a directory for topographic maps and create cogs for each. diff --git a/src/commands/basemaps-topo-import/types/tiff-item.ts b/src/commands/basemaps-topo-import/types/tiff-item.ts index 7cde6d65..a3ca8907 100644 --- a/src/commands/basemaps-topo-import/types/tiff-item.ts +++ b/src/commands/basemaps-topo-import/types/tiff-item.ts @@ -1,4 +1,4 @@ -import { Bounds, Epsg } from '@basemaps/geo'; +import { Bounds, Epsg, Size } from '@basemaps/geo'; import { Tiff } from '@basemaps/shared'; export class TiffItem { @@ -8,14 +8,16 @@ export class TiffItem { readonly version: string; readonly bounds: Bounds; readonly epsg: Epsg; + readonly size: Size; - constructor(tiff: Tiff, source: URL, mapCode: string, version: string, bounds: Bounds, epsg: Epsg) { + constructor(tiff: Tiff, source: URL, mapCode: string, version: string, bounds: Bounds, epsg: Epsg, size: Size) { this.tiff = tiff; this.source = source; this.mapCode = mapCode; this.version = version; this.bounds = bounds; this.epsg = epsg; + this.size = size; } toJSON(): string { From db4fab3ee7fecb30eab00d7f4f6004c6829b22ed Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 11 Dec 2024 11:47:45 +1300 Subject: [PATCH 34/47] Don't error if no size --- .../basemaps-topo-import/extractors/extract-bounds.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts index 103b699f..4d2f412d 100644 --- a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts +++ b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts @@ -31,7 +31,11 @@ export async function extractBounds(tiff: Tiff): Promise { * @returns if succeeded, a Bounds object. Otherwise, null. */ export function extractSize(tiff: Tiff): Size | null { - const size = tiff.images[0]?.size; - if (size == null || size.width == null || size.height == null) return null; - return size; + try { + const size = tiff.images[0]?.size ?? null; + return size; + } catch (e) { + logger.info({ found: false }, 'extractSize()'); + return null; + } } From 849f52ea155d2bf4223c27bc78c8377479f66538 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Mon, 16 Dec 2024 16:13:41 +1300 Subject: [PATCH 35/47] refactor: remove unused gdal command options --- .../gdal/gdal-build-cog.ts | 22 +++++------ .../gdal/gdal-build-vrt-warp.ts | 29 ++------------ .../gdal/gdal-build-vrt.ts | 38 ++----------------- .../basemaps-topo-import/gdal/gdal-opts.ts | 6 --- .../basemaps-topo-import/topo-cog-creation.ts | 27 +++---------- 5 files changed, 23 insertions(+), 99 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/gdal/gdal-opts.ts diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts index 44e9b567..7739eee0 100644 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts @@ -1,5 +1,4 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; import { urlToString } from '../../common.js'; @@ -7,12 +6,13 @@ export const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; interface gdalBuildCogOptions { /** - * Resampling algorithm. Nearest is the default. + * The number of pixels to remove from the right side of the imagery. + * + * If the value is a decimal number ending in above .5, the number of + * pixels removed will be the value rounded up to the nearest integer. + * The imagery will then be scaled to fulfil the difference. * - * @link - * https://gdal.org/en/stable/programs/gdal_translate.html#cmdoption-gdal_translate-r */ - resamplingMethod?: GdalResampling; pixelTrim?: number; } @@ -32,6 +32,7 @@ export function gdalBuildCogCommands( opts?: gdalBuildCogOptions, ): GdalCommand { const pixelTrim = opts?.pixelTrim ?? DEFAULT_TRIM_PIXEL_RIGHT; + const command: GdalCommand = { output, command: 'gdal_translate', @@ -39,9 +40,9 @@ export function gdalBuildCogCommands( ['-q'], // Supress non-error output ['-stats'], // Force stats (re)computation ['-of', 'COG'], // Output format + ['-srcwin', '0', '0', `${width - pixelTrim}`, `${height}`], // https://gdal.org/en/latest/drivers/raster/cog.html#creation-options - ['-srcwin', `0`, `0`, `${width - pixelTrim}`, `${height}`], ['-co', 'BIGTIFF=NO'], ['-co', 'BLOCKSIZE=512'], ['-co', 'COMPRESS=WEBP'], @@ -55,17 +56,14 @@ export function gdalBuildCogCommands( // https://gdal.org/en/latest/drivers/raster/cog.html#reprojection-related-creation-options ['-co', 'ADD_ALPHA=YES'], + + urlToString(input), + urlToString(output), ] .filter((f) => f != null) .flat() .map(String), }; - if (opts?.resamplingMethod != null) { - command.args.push('-r', opts.resamplingMethod); - } - - command.args.push(urlToString(input), urlToString(output)); - return command; } diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts index 12f7b49b..d4ee448d 100644 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts @@ -1,19 +1,8 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; import { Epsg } from '@basemaps/geo'; import { urlToString } from '../../common.js'; -interface gdalBuildVrtWarpOptions { - /** - * Resampling algorithm. Nearest is the default. - * - * @link - * https://gdal.org/en/latest/programs/gdalwarp.html#cmdoption-gdalwarp-r - */ - resamplingMethod?: GdalResampling; -} - /** * Constructs a 'gdalwarp' GdalCommand. * @@ -23,12 +12,7 @@ interface gdalBuildVrtWarpOptions { * @param opts * @returns */ -export function gdalBuildVrtWarp( - targetVrt: URL, - sourceVrt: URL, - sourceProj: Epsg, - opts?: gdalBuildVrtWarpOptions, -): GdalCommand { +export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { const command: GdalCommand = { output: targetVrt, command: 'gdalwarp', @@ -37,19 +21,14 @@ export function gdalBuildVrtWarp( ['-of', 'vrt'], // Output as a VRT ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp ['-s_srs', sourceProj.toEpsgString()], // Source EPSG - // ['-t_srs', Nztm2000QuadTms.projection.toEpsgString()], // Target EPSG - // ['-tr', targetResolution, targetResolution], + + urlToString(sourceVrt), + urlToString(targetVrt), ] .filter((f) => f != null) .flat() .map(String), }; - if (opts?.resamplingMethod != null) { - command.args.push('-r', opts.resamplingMethod); - } - - command.args.push(urlToString(sourceVrt), urlToString(targetVrt)); - return command; } diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts index 55c195a8..09892191 100644 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts @@ -1,30 +1,7 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; import { urlToString } from '../../common.js'; -interface GdalBuildVrtOptions { - /** - * Resampling algorithm. Nearest is the default. - * - * @link - * https://gdal.org/en/latest/programs/gdalbuildvrt.html#cmdoption-gdalbuildvrt-r - */ - resamplingMethod?: GdalResampling; - - /** - * Background color. - * - * @example - * '208 231 244' // NZ Topo Raster sea colour (RGB as 'RRR GGG BBB') - * - * @links - * https://gdal.org/en/latest/programs/gdalbuildvrt.html#cmdoption-gdalbuildvrt-vrtnodata - * https://gdal.org/en/latest/programs/gdalbuildvrt.html#vrtnodata - */ - background?: string; -} - /** * Constructs a 'gdalBuildVrt' GdalCommand. * @@ -33,23 +10,14 @@ interface GdalBuildVrtOptions { * @param opts * @returns */ -export function gdalBuildVrt(targetVrt: URL, source: URL[], opts?: GdalBuildVrtOptions): GdalCommand { +export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { if (source.length === 0) throw new Error('No source files given for :' + targetVrt.href); + const command: GdalCommand = { output: targetVrt, command: 'gdalbuildvrt', - args: ['-addalpha'], + args: ['-addalpha', urlToString(targetVrt), ...source.map(urlToString)], }; - if (opts?.background != null) { - command.args.push('-hidenodata', '-vrtnodata', opts.background); - } - - if (opts?.resamplingMethod != null) { - command.args.push('-r', opts.resamplingMethod); - } - - command.args.push(urlToString(targetVrt), ...source.map(urlToString)); - return command; } diff --git a/src/commands/basemaps-topo-import/gdal/gdal-opts.ts b/src/commands/basemaps-topo-import/gdal/gdal-opts.ts deleted file mode 100644 index 13b105f6..00000000 --- a/src/commands/basemaps-topo-import/gdal/gdal-opts.ts +++ /dev/null @@ -1,6 +0,0 @@ -const resamplingOptions = ['nearest', 'bilinear', 'cubic', 'cubicspline', 'lanczos', 'average', 'mode'] as const; -type ResamplingOption = (typeof resamplingOptions)[number]; - -export function checkResamplingMethod(resamplingMethod: string): resamplingMethod is ResamplingOption { - return (resamplingOptions as readonly string[]).includes(resamplingMethod); -} diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index a18dffc2..5077b3a8 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -1,7 +1,7 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { createFileStats, GdalResampling } from '@basemaps/cogify/build/cogify/stac.js'; +import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; import { Epsg } from '@basemaps/geo'; import { fsa, Tiff } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; @@ -19,7 +19,6 @@ import { loadInput } from '../group/group.js'; import { gdalBuildCogCommands as gdalBuildCog } from './gdal/gdal-build-cog.js'; import { gdalBuildVrt } from './gdal/gdal-build-vrt.js'; import { gdalBuildVrtWarp } from './gdal/gdal-build-vrt-warp.js'; -import { checkResamplingMethod as isValidResamplingOption } from './gdal/gdal-opts.js'; const Q = pLimit(10); @@ -49,11 +48,6 @@ export const topoCogCreation = command({ long: 'from-file', description: 'JSON file to load inputs from, must be a JSON Array', }), - resamplingMethod: option({ - type: optional(string), - long: 'resampling-method', - description: 'the GDAL resampling algorithm to use', - }), pixelTrim: option({ type: optional(number), long: 'pixel-trim', @@ -65,15 +59,6 @@ export const topoCogCreation = command({ registerCli(this, args); logger.info('ListJobs:Start'); - // check if resampling method is a valid option - const resamplingMethod = args.resamplingMethod; - - if (resamplingMethod != null) { - if (!isValidResamplingOption(resamplingMethod)) { - throw new Error('the provided resampling method is not a valid option'); - } - } - // Load the items for processing const inputs: string[] = []; for (const input of args.inputs) inputs.push(...loadInput(input)); @@ -94,7 +79,7 @@ export const topoCogCreation = command({ await Promise.all( inputs.map((input) => Q(async () => { - await createCogs(tryParseUrl(input), tmpFolder, args.pixelTrim, resamplingMethod); + await createCogs(tryParseUrl(input), tmpFolder, args.pixelTrim); }), ), ); @@ -103,7 +88,7 @@ export const topoCogCreation = command({ }, }); -async function createCogs(input: URL, tmp: URL, pixelTrim?: number, resamplingMethod?: GdalResampling): Promise { +async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise { const startTime = performance.now(); const item = await fsa.readJson(input); const tmpFolder = new URL(item.id, tmp); @@ -142,7 +127,7 @@ async function createCogs(input: URL, tmp: URL, pixelTrim?: number, resamplingMe logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); - const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath], { resamplingMethod }); + const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath]); await new GdalRunner(commandBuildVrt).run(logger); @@ -158,7 +143,7 @@ async function createCogs(input: URL, tmp: URL, pixelTrim?: number, resamplingMe if (sourceProj == null) throw new Error(`Unknown source projection ${sourceEpsg}`); const vrtWarpPath = new URL(`${item.id}-warp.vrt`, tmpFolder); - const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj, { resamplingMethod }); + const commandBuildVrtWarp = gdalBuildVrtWarp(vrtWarpPath, vrtPath, sourceProj); await new GdalRunner(commandBuildVrtWarp).run(logger); @@ -170,7 +155,7 @@ async function createCogs(input: URL, tmp: URL, pixelTrim?: number, resamplingMe const width = Number(item.properties['source.width']); const height = Number(item.properties['source.height']); const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const commandTranslate = gdalBuildCog(vrtWarpPath, tempPath, width, height, { resamplingMethod, pixelTrim }); + const commandTranslate = gdalBuildCog(vrtWarpPath, tempPath, width, height, { pixelTrim }); await new GdalRunner(commandTranslate).run(logger); From 62141833c7c2e10abe2b576d699221f26264b6d6 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Tue, 17 Dec 2024 14:48:10 +1300 Subject: [PATCH 36/47] refactor: type safety for additional Stac Item properties --- .../extractors/extract-bounds-from-tiff.ts | 24 ++++++ .../extractors/extract-bounds.ts | 41 ----------- .../extract-map-code-and-version.ts | 2 +- .../extractors/extract-size-from-tiff.ts | 23 ++++++ .../mappers/group-tiffs-by-directory.ts | 7 +- .../stac/create-base-stac-item.ts | 10 +-- .../stac/create-stac-collection.ts | 7 +- .../stac/create-stac-item-groups.ts | 5 +- .../stac/write-stac-files.ts | 5 +- .../basemaps-topo-import/topo-cog-creation.ts | 22 +++--- .../topo-stac-creation.ts | 6 +- .../types/map-sheet-stac-item.ts | 73 +++++++++++++++++++ 12 files changed, 151 insertions(+), 74 deletions(-) create mode 100644 src/commands/basemaps-topo-import/extractors/extract-bounds-from-tiff.ts delete mode 100644 src/commands/basemaps-topo-import/extractors/extract-bounds.ts create mode 100644 src/commands/basemaps-topo-import/extractors/extract-size-from-tiff.ts create mode 100644 src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts diff --git a/src/commands/basemaps-topo-import/extractors/extract-bounds-from-tiff.ts b/src/commands/basemaps-topo-import/extractors/extract-bounds-from-tiff.ts new file mode 100644 index 00000000..618250b8 --- /dev/null +++ b/src/commands/basemaps-topo-import/extractors/extract-bounds-from-tiff.ts @@ -0,0 +1,24 @@ +import { Bounds } from '@basemaps/geo'; +import { Tiff } from '@cogeotiff/core'; + +import { logger } from '../../../log.js'; +import { findBoundingBox } from '../../../utils/geotiff.js'; + +/** + * Attempts to extract bounds from the given Tiff object. + * + * @param tiff: The Tiff object from which to extract bounds + * + * @returns if succeeded, a Bounds object. Otherwise, null. + */ +export async function extractBoundsFromTiff(tiff: Tiff): Promise { + try { + const bounds = Bounds.fromBbox(await findBoundingBox(tiff)); + + logger.info({ found: true }, 'extractBoundsFromTiff()'); + return bounds; + } catch (e) { + logger.info({ found: false }, 'extractBoundsFromTiff()'); + return null; + } +} diff --git a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts b/src/commands/basemaps-topo-import/extractors/extract-bounds.ts deleted file mode 100644 index 4d2f412d..00000000 --- a/src/commands/basemaps-topo-import/extractors/extract-bounds.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { Bounds, Size } from '@basemaps/geo'; -import { Tiff } from '@cogeotiff/core'; - -import { logger } from '../../../log.js'; -import { findBoundingBox } from '../../../utils/geotiff.js'; - -/** - * This function attempts to extract bounds from the given Tiff object. - * - * @param tiff: The Tiff object from which to extract bounds - * - * @returns if succeeded, a Bounds object. Otherwise, null. - */ -export async function extractBounds(tiff: Tiff): Promise { - try { - const bounds = Bounds.fromBbox(await findBoundingBox(tiff)); - - logger.info({ found: true }, 'extractBounds()'); - return bounds; - } catch (e) { - logger.info({ found: false }, 'extractBounds()'); - return null; - } -} - -/** - * This function attempts to extract bounds from the given Tiff object. - * - * @param tiff: The Tiff object from which to extract bounds - * - * @returns if succeeded, a Bounds object. Otherwise, null. - */ -export function extractSize(tiff: Tiff): Size | null { - try { - const size = tiff.images[0]?.size ?? null; - return size; - } catch (e) { - logger.info({ found: false }, 'extractSize()'); - return null; - } -} diff --git a/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts b/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts index 31e7e78c..cfea949a 100644 --- a/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts +++ b/src/commands/basemaps-topo-import/extractors/extract-map-code-and-version.ts @@ -24,7 +24,7 @@ export function extractMapCodeAndVersion(file: string): { mapCode: string; versi const mapCode = fileName.split('_')[0]; if (mapCode == null) throw new Error('Map sheet not found in the file name'); - // extract version from tail of the file name (e.g. v1-0) + // extract version from tail of the file name (e.g. v1-00) const version = fileName.match(/v(\d)-(\d\d)/)?.[0]; if (version == null) throw new Error('Version not found in the file name'); diff --git a/src/commands/basemaps-topo-import/extractors/extract-size-from-tiff.ts b/src/commands/basemaps-topo-import/extractors/extract-size-from-tiff.ts new file mode 100644 index 00000000..43218a4b --- /dev/null +++ b/src/commands/basemaps-topo-import/extractors/extract-size-from-tiff.ts @@ -0,0 +1,23 @@ +import { Size } from '@basemaps/geo'; +import { Tiff } from '@cogeotiff/core'; + +import { logger } from '../../../log.js'; + +/** + * Attempts to extract a size from the given Tiff object. + * + * @param tiff: The Tiff object from which to extract the size + * + * @returns if succeeded, a Size object. Otherwise, null. + */ +export function extractSizeFromTiff(tiff: Tiff): Size | null { + try { + const size = tiff.images[0]?.size ?? null; + + logger.info({ found: true }, 'extractSizeFromTiff()'); + return size; + } catch (e) { + logger.info({ found: false }, 'extractSizeFromTiff()'); + return null; + } +} diff --git a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts index a375a87a..006b9499 100644 --- a/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts +++ b/src/commands/basemaps-topo-import/mappers/group-tiffs-by-directory.ts @@ -1,9 +1,10 @@ import { Tiff } from '@cogeotiff/core'; import { logger } from '../../../log.js'; -import { extractBounds, extractSize } from '../extractors/extract-bounds.js'; +import { extractBoundsFromTiff as extractBoundsFromTiff } from '../extractors/extract-bounds-from-tiff.js'; import { extractEpsgFromTiff } from '../extractors/extract-epsg-from-tiff.js'; import { extractMapCodeAndVersion } from '../extractors/extract-map-code-and-version.js'; +import { extractSizeFromTiff as extractSizeFromTiff } from '../extractors/extract-size-from-tiff.js'; import { brokenTiffs } from '../topo-stac-creation.js'; import { ByDirectory } from '../types/by-directory.js'; import { TiffItem } from '../types/tiff-item.js'; @@ -25,9 +26,9 @@ export async function groupTiffsByDirectory(tiffs: Tiff[]): Promise { +): Promise<{ all: MapSheetStacItem[]; latest: MapSheetStacItem }> { const allStacItems = all.map((item) => createBaseStacItem(`${item.mapCode}_${item.version}`, item)); const latestURL = new URL(`${latest.mapCode}_${latest.version}.json`, allTargetURL); diff --git a/src/commands/basemaps-topo-import/stac/write-stac-files.ts b/src/commands/basemaps-topo-import/stac/write-stac-files.ts index c0942b0f..71baf083 100644 --- a/src/commands/basemaps-topo-import/stac/write-stac-files.ts +++ b/src/commands/basemaps-topo-import/stac/write-stac-files.ts @@ -1,11 +1,12 @@ import { fsa } from '@basemaps/shared'; -import { StacCollection, StacItem } from 'stac-ts'; +import { StacCollection } from 'stac-ts'; import { logger } from '../../../log.js'; +import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; export async function writeStacFiles( target: URL, - items: StacItem[], + items: MapSheetStacItem[], collection: StacCollection, ): Promise<{ itemPaths: URL[]; collectionPath: URL }> { // Create collection json for all topo50-latest items. diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 5077b3a8..bfa9a6e5 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -9,7 +9,6 @@ import { command, number, option, optional, restPositionals, string } from 'cmd- import { mkdir, rm } from 'fs/promises'; import pLimit from 'p-limit'; import path from 'path'; -import { StacItem } from 'stac-ts'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; @@ -19,6 +18,7 @@ import { loadInput } from '../group/group.js'; import { gdalBuildCogCommands as gdalBuildCog } from './gdal/gdal-build-cog.js'; import { gdalBuildVrt } from './gdal/gdal-build-vrt.js'; import { gdalBuildVrtWarp } from './gdal/gdal-build-vrt-warp.js'; +import { MapSheetStacItem } from './types/map-sheet-stac-item.js'; const Q = pLimit(10); @@ -90,29 +90,29 @@ export const topoCogCreation = command({ async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise { const startTime = performance.now(); - const item = await fsa.readJson(input); + const item = await fsa.readJson(input); const tmpFolder = new URL(item.id, tmp); try { // Extract the source URL from the item logger.info({ item: item.id }, 'CogCreation:Start'); - const source = item.assets['source']?.href; - if (source == null) throw new Error('No source file found in the item'); + const source = item.assets.source.href; await mkdir(tmpFolder, { recursive: true }); // Download the source file const sourceUrl = tryParseUrl(source); - const filePath = path.parse(sourceUrl.href); - const fileName = filePath.base; if (!(await fsa.exists(sourceUrl))) throw new Error('Source file not found'); const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); + const filePath = path.parse(sourceUrl.href); + const fileName = filePath.base; const inputPath = new URL(fileName, tmpFolder); + logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); // Add checksum for source file if (item.assets['source'] == null) throw new Error('No source file found in the item'); await fsa.write(inputPath, hashStreamSource); - item.assets['source']['file:checksum'] = hashStreamSource.multihash; item.assets['source']['file:size'] = hashStreamSource.size; + item.assets['source']['file:checksum'] = hashStreamSource.multihash; // read resolution from first image of tiff const tiff = await new Tiff(fsa.source(inputPath)).init(); @@ -137,8 +137,6 @@ async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise Date: Tue, 17 Dec 2024 14:57:45 +1300 Subject: [PATCH 37/47] Revert "Build on a dgal container" This reverts commit b5499e3f650b576a05167229c576088918e5628b. --- Dockerfile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0f17c4d2..1ce55934 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,6 @@ -FROM ghcr.io/osgeo/gdal:ubuntu-small-3.8.0 +FROM node:20-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 RUN apt-get update && apt-get install openssh-client git -y -RUN apt-get install -y ca-certificates curl gnupg -RUN mkdir -p /etc/apt/keyrings -RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -ENV NODE_MAJOR=20 -RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list - -RUN apt-get update -RUN apt-get install -y nodejs WORKDIR /app From ae4a30e509887dc5ed5a5ea18f0d11301b47d51b Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 18 Dec 2024 14:50:01 +1300 Subject: [PATCH 38/47] Give ulid for tmp job folder and remove gdalwrap --- .../gdal/gdal-build-cog.ts | 3 +- .../gdal/gdal-build-vrt-warp.ts | 34 ------------------- .../gdal/gdal-build-vrt.ts | 3 +- .../basemaps-topo-import/topo-cog-creation.ts | 22 +++--------- 4 files changed, 6 insertions(+), 56 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts index 7739eee0..46e418ed 100644 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts @@ -1,6 +1,5 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; - -import { urlToString } from '../../common.js'; +import { urlToString } from '@basemaps/shared'; export const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts deleted file mode 100644 index d4ee448d..00000000 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt-warp.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { Epsg } from '@basemaps/geo'; - -import { urlToString } from '../../common.js'; - -/** - * Constructs a 'gdalwarp' GdalCommand. - * - * @param targetVrt - * @param sourceVrt - * @param sourceProj - * @param opts - * @returns - */ -export function gdalBuildVrtWarp(targetVrt: URL, sourceVrt: URL, sourceProj: Epsg): GdalCommand { - const command: GdalCommand = { - output: targetVrt, - command: 'gdalwarp', - args: [ - ['-multi'], // Mutithread IO - ['-of', 'vrt'], // Output as a VRT - ['-wo', 'NUM_THREADS=ALL_CPUS'], // Multithread the warp - ['-s_srs', sourceProj.toEpsgString()], // Source EPSG - - urlToString(sourceVrt), - urlToString(targetVrt), - ] - .filter((f) => f != null) - .flat() - .map(String), - }; - - return command; -} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts index 09892191..d9ac82ef 100644 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts +++ b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts @@ -1,6 +1,5 @@ import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; - -import { urlToString } from '../../common.js'; +import { urlToString } from '@basemaps/shared'; /** * Constructs a 'gdalBuildVrt' GdalCommand. diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index bfa9a6e5..58b12bca 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -2,13 +2,13 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; -import { Epsg } from '@basemaps/geo'; import { fsa, Tiff } from '@basemaps/shared'; import { CliId } from '@basemaps/shared/build/cli/info.js'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; import { mkdir, rm } from 'fs/promises'; import pLimit from 'p-limit'; import path from 'path'; +import ulid from 'ulid'; import { CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; @@ -17,7 +17,6 @@ import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../commo import { loadInput } from '../group/group.js'; import { gdalBuildCogCommands as gdalBuildCog } from './gdal/gdal-build-cog.js'; import { gdalBuildVrt } from './gdal/gdal-build-vrt.js'; -import { gdalBuildVrtWarp } from './gdal/gdal-build-vrt-warp.js'; import { MapSheetStacItem } from './types/map-sheet-stac-item.js'; const Q = pLimit(10); @@ -91,7 +90,8 @@ export const topoCogCreation = command({ async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise { const startTime = performance.now(); const item = await fsa.readJson(input); - const tmpFolder = new URL(item.id, tmp); + const jobId = ulid.ulid(); + const tmpFolder = new URL(jobId, tmp); try { // Extract the source URL from the item logger.info({ item: item.id }, 'CogCreation:Start'); @@ -131,20 +131,6 @@ async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise Date: Wed, 18 Dec 2024 15:12:59 +1300 Subject: [PATCH 39/47] Fix the path of temp folder --- src/commands/basemaps-topo-import/topo-cog-creation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index 58b12bca..ad3d0760 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -91,7 +91,7 @@ async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise(input); const jobId = ulid.ulid(); - const tmpFolder = new URL(jobId, tmp); + const tmpFolder = new URL(`${jobId}/`, tmp); try { // Extract the source URL from the item logger.info({ item: item.id }, 'CogCreation:Start'); From daa9c08026246c761bc6923307f9bd8386f8d3bc Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 18 Dec 2024 15:56:42 +1300 Subject: [PATCH 40/47] Add docker.io to the container --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1ce55934..a1164701 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM node:20-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 -RUN apt-get update && apt-get install openssh-client git -y +RUN apt-get update && apt-get install openssh-client git docker.io -y WORKDIR /app From da8db627d8d0aed9dfae04e102a123744d69bce0 Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Wed, 8 Jan 2025 10:41:22 +1300 Subject: [PATCH 41/47] feature: append basemaps options to generated Stac Item files --- .../stac/create-base-stac-item.ts | 20 +++++++++++++++++-- .../types/map-sheet-stac-item.ts | 5 +++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts index 59d8df88..a1f2fa7f 100644 --- a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -5,7 +5,8 @@ import { logger } from '../../../log.js'; import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; import { TiffItem } from '../types/tiff-item.js'; -const cliDate = new Date().toISOString(); +const CLI_DATE = new Date().toISOString(); +const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; /** * This function creates a base StacItem object based on the provided parameters. @@ -38,12 +39,27 @@ export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapShe }, stac_extensions: ['https://stac-extensions.github.io/file/v2.0.0/schema.json'], properties: { - datetime: cliDate, + datetime: CLI_DATE, map_code: tiffItem.mapCode, version: tiffItem.version.replace('-', '.'), // e.g. "v1-00" to "v1.00" 'proj:epsg': tiffItem.epsg.code, 'source.width': tiffItem.size.width, 'source.height': tiffItem.size.height, + 'linz_basemaps:options': { + preset: 'webp', + blockSize: 512, + bigTIFF: 'no', + compression: 'webp', + quality: 100, + overviewCompress: 'webp', + overviewQuality: 90, + overviewResampling: 'lanczos', + tileId: fileName, + sourceEpsg: tiffItem.epsg.code, + addalpha: true, + noReprojecting: true, + srcwin: [0, 0, tiffItem.size.width - DEFAULT_TRIM_PIXEL_RIGHT, tiffItem.size.height], + }, }, geometry: { type: 'Polygon', coordinates: tiffItem.bounds.toPolygon() } as GeoJSONPolygon, bbox: tiffItem.bounds.toBbox(), diff --git a/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts b/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts index f8ca45b2..cd596298 100644 --- a/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts +++ b/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts @@ -69,5 +69,10 @@ export interface MapSheetStacItem extends StacItem { * The height of a map sheet in pixels. */ 'source.height': number; + + /** + * An object of key-value pairs describing options for Basemaps' cogify process. + */ + 'linz_basemaps:options': { [key: string]: unknown }; }; } From 35c73436f236548992f312a70f59a1893a202863 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Wed, 8 Jan 2025 14:45:21 +1300 Subject: [PATCH 42/47] Add more information to the stac item --- src/cli.info.ts | 5 +++++ .../stac/create-base-stac-item.ts | 13 +++++++++++-- .../stac/create-stac-collection.ts | 2 +- .../basemaps-topo-import/topo-cog-creation.ts | 3 +-- .../types/map-sheet-stac-item.ts | 5 +++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/cli.info.ts b/src/cli.info.ts index b82025fc..f45816a1 100644 --- a/src/cli.info.ts +++ b/src/cli.info.ts @@ -1,3 +1,5 @@ +import * as ulid from 'ulid'; + export const CliInfo = { package: '@linzjs/argo-tasks', // Git version information @@ -7,3 +9,6 @@ export const CliInfo = { // Github action that the CLI was built from buildId: process.env['GITHUB_RUN_ID'] ? `${process.env['GITHUB_RUN_ID']}-${process.env['GITHUB_RUN_ATTEMPT']}` : '', }; + +/** Unique Id for this instance of the cli being run */ +export const CliId = ulid.ulid(); diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts index a1f2fa7f..7335090a 100644 --- a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -1,6 +1,7 @@ -import { CliId } from '@basemaps/shared/build/cli/info.js'; +import { Nztm2000Tms } from '@basemaps/geo'; import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; +import { CliId, CliInfo } from '../../../cli.info.js'; import { logger } from '../../../log.js'; import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; import { TiffItem } from '../types/tiff-item.js'; @@ -29,6 +30,7 @@ export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapShe { rel: 'self', href: `./${fileName}.json`, type: 'application/json' }, { rel: 'collection', href: './collection.json', type: 'application/json' }, { rel: 'parent', href: './collection.json', type: 'application/json' }, + { rel: 'linz_basemaps:source', href: tiffItem.source.href, type: 'image/tiff; application=geotiff' }, ], assets: { source: { @@ -46,6 +48,8 @@ export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapShe 'source.width': tiffItem.size.width, 'source.height': tiffItem.size.height, 'linz_basemaps:options': { + tileId: fileName, + tileMatrix: Nztm2000Tms.identifier, // TODO: We will need to get the tileMatrix by epsg in future once we support these tileMatrix - > TileMatrixSets.get(tiffItem.epsg).identifier, preset: 'webp', blockSize: 512, bigTIFF: 'no', @@ -54,12 +58,17 @@ export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapShe overviewCompress: 'webp', overviewQuality: 90, overviewResampling: 'lanczos', - tileId: fileName, sourceEpsg: tiffItem.epsg.code, addalpha: true, noReprojecting: true, srcwin: [0, 0, tiffItem.size.width - DEFAULT_TRIM_PIXEL_RIGHT, tiffItem.size.height], }, + 'linz_basemaps:generated': { + package: CliInfo.package, + hash: CliInfo.hash, + version: CliInfo.version, + datetime: CLI_DATE, + }, }, geometry: { type: 'Polygon', coordinates: tiffItem.bounds.toPolygon() } as GeoJSONPolygon, bbox: tiffItem.bounds.toBbox(), diff --git a/src/commands/basemaps-topo-import/stac/create-stac-collection.ts b/src/commands/basemaps-topo-import/stac/create-stac-collection.ts index 482ba7a3..b8e1b74a 100644 --- a/src/commands/basemaps-topo-import/stac/create-stac-collection.ts +++ b/src/commands/basemaps-topo-import/stac/create-stac-collection.ts @@ -1,7 +1,7 @@ import { Bounds } from '@basemaps/geo'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; import { StacCollection } from 'stac-ts'; +import { CliId } from '../../../cli.info.js'; import { logger } from '../../../log.js'; import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts index ad3d0760..18171f58 100644 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ b/src/commands/basemaps-topo-import/topo-cog-creation.ts @@ -3,14 +3,13 @@ import { tmpdir } from 'node:os'; import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; import { fsa, Tiff } from '@basemaps/shared'; -import { CliId } from '@basemaps/shared/build/cli/info.js'; import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; import { mkdir, rm } from 'fs/promises'; import pLimit from 'p-limit'; import path from 'path'; import ulid from 'ulid'; -import { CliInfo } from '../../cli.info.js'; +import { CliId, CliInfo } from '../../cli.info.js'; import { logger } from '../../log.js'; import { HashTransform } from '../../utils/hash.stream.js'; import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; diff --git a/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts b/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts index cd596298..17197799 100644 --- a/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts +++ b/src/commands/basemaps-topo-import/types/map-sheet-stac-item.ts @@ -74,5 +74,10 @@ export interface MapSheetStacItem extends StacItem { * An object of key-value pairs describing options for Basemaps' cogify process. */ 'linz_basemaps:options': { [key: string]: unknown }; + + /** + * An object of key-value pairs information for basemaps cli packages. + */ + 'linz_basemaps:generated': { [key: string]: unknown }; }; } From 34373a2b7c45fa36cefc53a5f1185da6c46bd980 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Thu, 9 Jan 2025 11:14:04 +1300 Subject: [PATCH 43/47] Update the tile.json structure for cogify to parse --- src/commands/basemaps-topo-import/stac/write-stac-files.ts | 6 +++--- src/commands/basemaps-topo-import/topo-stac-creation.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/commands/basemaps-topo-import/stac/write-stac-files.ts b/src/commands/basemaps-topo-import/stac/write-stac-files.ts index 71baf083..3ec919d9 100644 --- a/src/commands/basemaps-topo-import/stac/write-stac-files.ts +++ b/src/commands/basemaps-topo-import/stac/write-stac-files.ts @@ -8,16 +8,16 @@ export async function writeStacFiles( target: URL, items: MapSheetStacItem[], collection: StacCollection, -): Promise<{ itemPaths: URL[]; collectionPath: URL }> { +): Promise<{ itemPaths: { path: URL }[]; collectionPath: URL }> { // Create collection json for all topo50-latest items. logger.info({ target }, 'CreateStac:Output'); logger.info({ items: items.length, collectionID: collection.id }, 'Stac:Output'); - const itemPaths: URL[] = []; + const itemPaths = []; for (const item of items) { const itemPath = new URL(`${item.id}.json`, target); - itemPaths.push(itemPath); + itemPaths.push({ path: itemPath }); await fsa.write(itemPath, JSON.stringify(item, null, 2)); } diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index 7101a64a..b273dd3a 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -120,7 +120,7 @@ async function loadTiffsToCreateStacs( force: boolean, scale: string, resolution: string, -): Promise<{ epsgDirectoryPaths: { epsg: string; url: URL }[]; stacItemPaths: URL[] }> { +): Promise<{ epsgDirectoryPaths: { epsg: string; url: URL }[]; stacItemPaths: { path: URL }[] }> { logger.info({ source }, 'LoadTiffs:Start'); // extract all file paths from the source directory and convert them into URL objects const fileURLs = await fsa.toArray(fsa.list(source)); @@ -136,7 +136,7 @@ async function loadTiffsToCreateStacs( logger.info('GroupTiffs:End'); const epsgDirectoryPaths: { epsg: string; url: URL }[] = []; - const stacItemPaths: URL[] = []; + const stacItemPaths = []; // create and write stac items and collections for (const [epsg, itemsByMapCode] of itemsByDir.all.entries()) { From 7b8b8398f8ec33ea958bf161dd51bf8e269a9358 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Fri, 10 Jan 2025 11:08:17 +1300 Subject: [PATCH 44/47] Remove tileMatrix --- src/commands/basemaps-topo-import/stac/create-base-stac-item.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts index 7335090a..318aa25b 100644 --- a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -1,4 +1,3 @@ -import { Nztm2000Tms } from '@basemaps/geo'; import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; import { CliId, CliInfo } from '../../../cli.info.js'; @@ -49,7 +48,6 @@ export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapShe 'source.height': tiffItem.size.height, 'linz_basemaps:options': { tileId: fileName, - tileMatrix: Nztm2000Tms.identifier, // TODO: We will need to get the tileMatrix by epsg in future once we support these tileMatrix - > TileMatrixSets.get(tiffItem.epsg).identifier, preset: 'webp', blockSize: 512, bigTIFF: 'no', From 75dc131c674e9f6b0f9be6de86475d9221b6c8ef Mon Sep 17 00:00:00 2001 From: Tawera Manaena Date: Fri, 10 Jan 2025 12:44:00 +1300 Subject: [PATCH 45/47] feat: append slug and tile matrix metadata to generated Stac Item files --- .../mappers/map-epsg-to-slug.ts | 27 +++++++++++++++++++ .../stac/create-base-stac-item.ts | 4 ++- .../stac/create-stac-collection.ts | 9 +++++-- .../stac/create-stac-item-groups.ts | 15 ++++++----- .../topo-stac-creation.ts | 23 +++++++++++++--- 5 files changed, 65 insertions(+), 13 deletions(-) create mode 100644 src/commands/basemaps-topo-import/mappers/map-epsg-to-slug.ts diff --git a/src/commands/basemaps-topo-import/mappers/map-epsg-to-slug.ts b/src/commands/basemaps-topo-import/mappers/map-epsg-to-slug.ts new file mode 100644 index 00000000..f0e7dd68 --- /dev/null +++ b/src/commands/basemaps-topo-import/mappers/map-epsg-to-slug.ts @@ -0,0 +1,27 @@ +import { EpsgCode } from '@basemaps/geo'; + +import { logger } from '../../../log.js'; + +const slugs: { [key in EpsgCode]?: string } = { + [EpsgCode.Nztm2000]: 'new-zealand-mainland', + [EpsgCode.Citm2000]: 'chatham-islands', +}; + +/** + * Attempts to map the given EpsgCode enum to a slug. + * + * @param epsg: The EpsgCode enum to map to a slug + * + * @returns if succeeded, a slug string. Otherwise, null. + */ +export function mapEpsgToSlug(epsg: EpsgCode): string | null { + const slug = slugs[epsg]; + + if (slug == null) { + logger.info({ found: false }, 'mapEpsgToSlug()'); + return null; + } + + logger.info({ found: true }, 'mapEpsgToSlug()'); + return slug; +} diff --git a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts index 318aa25b..bb8827f0 100644 --- a/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts +++ b/src/commands/basemaps-topo-import/stac/create-base-stac-item.ts @@ -1,3 +1,4 @@ +import { TileMatrixSet } from '@basemaps/geo'; import { GeoJSONPolygon } from 'stac-ts/src/types/geojson.js'; import { CliId, CliInfo } from '../../../cli.info.js'; @@ -18,7 +19,7 @@ const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; * * @returns a StacItem object */ -export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapSheetStacItem { +export function createBaseStacItem(fileName: string, tiffItem: TiffItem, tileMatrix: TileMatrixSet): MapSheetStacItem { logger.info({ fileName }, 'createBaseStacItem()'); const item: MapSheetStacItem = { @@ -48,6 +49,7 @@ export function createBaseStacItem(fileName: string, tiffItem: TiffItem): MapShe 'source.height': tiffItem.size.height, 'linz_basemaps:options': { tileId: fileName, + tileMatrix: tileMatrix.identifier, preset: 'webp', blockSize: 512, bigTIFF: 'no', diff --git a/src/commands/basemaps-topo-import/stac/create-stac-collection.ts b/src/commands/basemaps-topo-import/stac/create-stac-collection.ts index b8e1b74a..c2d6c857 100644 --- a/src/commands/basemaps-topo-import/stac/create-stac-collection.ts +++ b/src/commands/basemaps-topo-import/stac/create-stac-collection.ts @@ -7,7 +7,12 @@ import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; const cliDate = new Date().toISOString(); -export function createStacCollection(title: string, imageryBounds: Bounds, items: MapSheetStacItem[]): StacCollection { +export function createStacCollection( + title: string, + linzSlug: string, + imageryBounds: Bounds, + items: MapSheetStacItem[], +): StacCollection { logger.info({ items: items.length }, 'CreateStacCollection()'); const collection: StacCollection = { type: 'Collection', @@ -37,7 +42,7 @@ export function createStacCollection(title: string, imageryBounds: Bounds, items 'linz:geospatial_category': 'topographic-maps', 'linz:region': 'new-zealand', 'linz:security_classification': 'unclassified', - 'linz:slug': 'topo50', + 'linz:slug': linzSlug, extent: { spatial: { bbox: [imageryBounds.toBbox()] }, // Default the temporal time today if no times were found as it is required for STAC diff --git a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts index a67bd9d0..1ec8bf17 100644 --- a/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts +++ b/src/commands/basemaps-topo-import/stac/create-stac-item-groups.ts @@ -1,21 +1,24 @@ +import { TileMatrixSet } from '@basemaps/geo'; + import { MapSheetStacItem } from '../types/map-sheet-stac-item.js'; import { TiffItem } from '../types/tiff-item.js'; import { createBaseStacItem } from './create-base-stac-item.js'; /** * This function needs to create two groups: - * - StacItem objects that will live in the "topo[50/250]" directory - * - StacItem objects that will live in the "topo[50/250]-latest" directory + * - StacItem objects that will live in the "topo[50|250]" directory + * - StacItem objects that will live in the "topo[50|250]_latest" directory * - * All versions need a StacItem object that lives in the topo[50/250] directory - * The latest version needs a second StacItem object that lives in the topo[50/250]-latest dir + * All versions need a StacItem object that will live in the topo[50/250] directory + * The latest version needs a second StacItem object that will live in the topo[50|250]_latest directory */ export async function createStacItems( allTargetURL: URL, + tileMatrix: TileMatrixSet, all: TiffItem[], latest: TiffItem, ): Promise<{ all: MapSheetStacItem[]; latest: MapSheetStacItem }> { - const allStacItems = all.map((item) => createBaseStacItem(`${item.mapCode}_${item.version}`, item)); + const allStacItems = all.map((item) => createBaseStacItem(`${item.mapCode}_${item.version}`, item, tileMatrix)); const latestURL = new URL(`${latest.mapCode}_${latest.version}.json`, allTargetURL); @@ -28,7 +31,7 @@ export async function createStacItems( }); }); - const latestStacItem = createBaseStacItem(latest.mapCode, latest); + const latestStacItem = createBaseStacItem(latest.mapCode, latest, tileMatrix); // add link to the latest item referencing its copy that will live in the topo[50/250] directory latestStacItem.links.push({ diff --git a/src/commands/basemaps-topo-import/topo-stac-creation.ts b/src/commands/basemaps-topo-import/topo-stac-creation.ts index b273dd3a..b11580f6 100644 --- a/src/commands/basemaps-topo-import/topo-stac-creation.ts +++ b/src/commands/basemaps-topo-import/topo-stac-creation.ts @@ -1,5 +1,5 @@ import { loadTiffsFromPaths } from '@basemaps/config-loader/build//json/tiff.config.js'; -import { Bounds } from '@basemaps/geo'; +import { Bounds, Epsg, Nztm2000Tms, TileMatrixSets } from '@basemaps/geo'; import { fsa } from '@basemaps/shared'; import { boolean, command, flag, option, string } from 'cmd-ts'; import pLimit from 'p-limit'; @@ -9,6 +9,7 @@ import { logger } from '../../log.js'; import { isArgo } from '../../utils/argo.js'; import { config, forceOutput, registerCli, tryParseUrl, UrlFolder, verbose } from '../common.js'; import { groupTiffsByDirectory } from './mappers/group-tiffs-by-directory.js'; +import { mapEpsgToSlug } from './mappers/map-epsg-to-slug.js'; import { createStacCollection } from './stac/create-stac-collection.js'; import { createStacItems } from './stac/create-stac-item-groups.js'; import { writeStacFiles } from './stac/write-stac-files.js'; @@ -149,6 +150,14 @@ async function loadTiffsToCreateStacs( const latestBounds: Bounds[] = []; const latestStacItems: MapSheetStacItem[] = []; + // parse epsg + const epsgCode = Epsg.parse(epsg); + if (epsgCode == null) throw new Error(`Failed to parse epsg '${epsg}'`); + + // convert epsg to tile matrix + const tileMatrix = TileMatrixSets.tryGet(epsgCode) ?? Nztm2000Tms; // TODO: support other tile matrices + if (tileMatrix == null) throw new Error(`Failed to convert epsg code '${epsgCode.code}' to a tile matrix`); + // create stac items logger.info({ epsg }, 'CreateStacItems:Start'); for (const [mapCode, items] of itemsByMapCode.entries()) { @@ -156,7 +165,7 @@ async function loadTiffsToCreateStacs( const latest = itemsByDir.latest.get(epsg).get(mapCode); // create stac items - const stacItems = await createStacItems(allTargetURL, items, latest); + const stacItems = await createStacItems(allTargetURL, tileMatrix, items, latest); allBounds.push(...items.map((item) => item.bounds)); allStacItems.push(...stacItems.all); @@ -165,9 +174,15 @@ async function loadTiffsToCreateStacs( latestStacItems.push(stacItems.latest); } + // convert epsg to slug + const epsgSlug = mapEpsgToSlug(epsgCode.code); + if (epsgSlug == null) throw new Error(`Failed to map epsg code '${epsgCode.code}' to a slug`); + + const linzSlug = `${scale}-${epsgSlug}`; + // create collections - const collection = createStacCollection(title, Bounds.union(allBounds), allStacItems); - const latestCollection = createStacCollection(title, Bounds.union(latestBounds), latestStacItems); + const collection = createStacCollection(title, linzSlug, Bounds.union(allBounds), allStacItems); + const latestCollection = createStacCollection(title, linzSlug, Bounds.union(latestBounds), latestStacItems); logger.info({ epsg }, 'CreateStacItems:End'); if (force || isArgo()) { From 32edd5c7d31ee01f9944adcb5689a5d1f6f2c2bb Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Mon, 13 Jan 2025 10:21:54 +1300 Subject: [PATCH 46/47] Remove topo-cog-creation cli and use cogify creat cog instead. --- package-lock.json | 12 -- package.json | 1 - .../gdal/gdal-build-cog.ts | 68 ------- .../gdal/gdal-build-vrt.ts | 22 --- .../basemaps-topo-import/topo-cog-creation.ts | 166 ------------------ src/commands/index.ts | 2 - 6 files changed, 271 deletions(-) delete mode 100644 src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts delete mode 100644 src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts delete mode 100644 src/commands/basemaps-topo-import/topo-cog-creation.ts diff --git a/package-lock.json b/package-lock.json index 035726ef..e0f062b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,7 +12,6 @@ "@aws-sdk/client-s3": "^3.440.0", "@aws-sdk/credential-providers": "^3.438.0", "@aws-sdk/lib-storage": "^3.440.0", - "@basemaps/cogify": "^7.12.0", "@basemaps/config": "^7.7.0", "@basemaps/config-loader": "^7.12.0", "@basemaps/geo": "^7.5.0", @@ -4434,17 +4433,6 @@ "node": ">=16.0.0" } }, - "node_modules/@basemaps/cogify": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@basemaps/cogify/-/cogify-7.12.0.tgz", - "integrity": "sha512-mkTKqWcp/2FoDANFLDyDCd7dFmN3OalfEoQ2QzpP5Z2FhYcWj9RXKc75aElm2fBTHELFKmvskT0B3rDIN30tTg==", - "bin": { - "cogify": "build/bin.js" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - } - }, "node_modules/@basemaps/config": { "version": "7.12.0", "resolved": "https://registry.npmjs.org/@basemaps/config/-/config-7.12.0.tgz", diff --git a/package.json b/package.json index 652aa288..cbf526e7 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,6 @@ "@aws-sdk/client-s3": "^3.440.0", "@aws-sdk/credential-providers": "^3.438.0", "@aws-sdk/lib-storage": "^3.440.0", - "@basemaps/cogify": "^7.12.0", "@basemaps/config": "^7.7.0", "@basemaps/config-loader": "^7.12.0", "@basemaps/geo": "^7.5.0", diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts deleted file mode 100644 index 46e418ed..00000000 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-cog.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { urlToString } from '@basemaps/shared'; - -export const DEFAULT_TRIM_PIXEL_RIGHT = 1.7; - -interface gdalBuildCogOptions { - /** - * The number of pixels to remove from the right side of the imagery. - * - * If the value is a decimal number ending in above .5, the number of - * pixels removed will be the value rounded up to the nearest integer. - * The imagery will then be scaled to fulfil the difference. - * - */ - pixelTrim?: number; -} - -/** - * Constructs a 'gdal_translate' GdalCommand. - * - * @param input - * @param output - * @param opts - * @returns - */ -export function gdalBuildCogCommands( - input: URL, - output: URL, - width: number, - height: number, - opts?: gdalBuildCogOptions, -): GdalCommand { - const pixelTrim = opts?.pixelTrim ?? DEFAULT_TRIM_PIXEL_RIGHT; - - const command: GdalCommand = { - output, - command: 'gdal_translate', - args: [ - ['-q'], // Supress non-error output - ['-stats'], // Force stats (re)computation - ['-of', 'COG'], // Output format - ['-srcwin', '0', '0', `${width - pixelTrim}`, `${height}`], - - // https://gdal.org/en/latest/drivers/raster/cog.html#creation-options - ['-co', 'BIGTIFF=NO'], - ['-co', 'BLOCKSIZE=512'], - ['-co', 'COMPRESS=WEBP'], - ['-co', 'NUM_THREADS=ALL_CPUS'], // Use all CPUS - ['-co', 'OVERVIEW_COMPRESS=WEBP'], - ['-co', 'OVERVIEWS=IGNORE_EXISTING'], - ['-co', 'OVERVIEW_QUALITY=90'], - ['-co', 'OVERVIEW_RESAMPLING=LANCZOS'], - ['-co', 'QUALITY=100'], - ['-co', 'SPARSE_OK=TRUE'], // Allow for sparse writes - - // https://gdal.org/en/latest/drivers/raster/cog.html#reprojection-related-creation-options - ['-co', 'ADD_ALPHA=YES'], - - urlToString(input), - urlToString(output), - ] - .filter((f) => f != null) - .flat() - .map(String), - }; - - return command; -} diff --git a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts b/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts deleted file mode 100644 index d9ac82ef..00000000 --- a/src/commands/basemaps-topo-import/gdal/gdal-build-vrt.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { GdalCommand } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { urlToString } from '@basemaps/shared'; - -/** - * Constructs a 'gdalBuildVrt' GdalCommand. - * - * @param targetVrt - * @param source - * @param opts - * @returns - */ -export function gdalBuildVrt(targetVrt: URL, source: URL[]): GdalCommand { - if (source.length === 0) throw new Error('No source files given for :' + targetVrt.href); - - const command: GdalCommand = { - output: targetVrt, - command: 'gdalbuildvrt', - args: ['-addalpha', urlToString(targetVrt), ...source.map(urlToString)], - }; - - return command; -} diff --git a/src/commands/basemaps-topo-import/topo-cog-creation.ts b/src/commands/basemaps-topo-import/topo-cog-creation.ts deleted file mode 100644 index 18171f58..00000000 --- a/src/commands/basemaps-topo-import/topo-cog-creation.ts +++ /dev/null @@ -1,166 +0,0 @@ -import { tmpdir } from 'node:os'; - -import { GdalRunner } from '@basemaps/cogify/build/cogify/gdal.runner.js'; -import { createFileStats } from '@basemaps/cogify/build/cogify/stac.js'; -import { fsa, Tiff } from '@basemaps/shared'; -import { command, number, option, optional, restPositionals, string } from 'cmd-ts'; -import { mkdir, rm } from 'fs/promises'; -import pLimit from 'p-limit'; -import path from 'path'; -import ulid from 'ulid'; - -import { CliId, CliInfo } from '../../cli.info.js'; -import { logger } from '../../log.js'; -import { HashTransform } from '../../utils/hash.stream.js'; -import { config, forceOutput, registerCli, tryParseUrl, verbose } from '../common.js'; -import { loadInput } from '../group/group.js'; -import { gdalBuildCogCommands as gdalBuildCog } from './gdal/gdal-build-cog.js'; -import { gdalBuildVrt } from './gdal/gdal-build-vrt.js'; -import { MapSheetStacItem } from './types/map-sheet-stac-item.js'; - -const Q = pLimit(10); - -/** - * List all the tiffs in a directory for topographic maps and create cogs for each. - * - * @param source: Location of the source files - * @example s3://linz-topographic-upload/topographic/TopoReleaseArchive/NZTopo50_GeoTif_Gridless/ - * - * @param target: Location of the target path - */ -export const topoCogCreation = command({ - name: 'topo-cog-creation', - description: 'Get the list of topo cog stac items and creating cog for them.', - version: CliInfo.version, - args: { - config, - verbose, - forceOutput, - inputs: restPositionals({ - type: string, - displayName: 'items', - description: 'list of items to group, can be a JSON array', - }), - fromFile: option({ - type: optional(string), - long: 'from-file', - description: 'JSON file to load inputs from, must be a JSON Array', - }), - pixelTrim: option({ - type: optional(number), - long: 'pixel-trim', - description: 'number of pixels to trim from the right side of the image', - }), - }, - async handler(args) { - const startTime = performance.now(); - registerCli(this, args); - logger.info('ListJobs:Start'); - - // Load the items for processing - const inputs: string[] = []; - for (const input of args.inputs) inputs.push(...loadInput(input)); - if (args.fromFile && (await fsa.exists(tryParseUrl(args.fromFile)))) { - const input = await fsa.readJson(tryParseUrl(args.fromFile)); - if (Array.isArray(input)) inputs.push(...input); - } - - if (inputs.length === 0) { - logger.error('Group:Error:Empty'); - process.exit(1); - } - - // Prepare temporary folder for dgdal to create cog - const tmpPath = path.join(tmpdir(), CliId); - const tmpURL = tryParseUrl(tmpPath); - const tmpFolder = tmpURL.href.endsWith('/') ? new URL(tmpURL.href) : new URL(`${tmpURL.href}/`); - await Promise.all( - inputs.map((input) => - Q(async () => { - await createCogs(tryParseUrl(input), tmpFolder, args.pixelTrim); - }), - ), - ); - - logger.info({ duration: performance.now() - startTime }, 'ListJobs:Done'); - }, -}); - -async function createCogs(input: URL, tmp: URL, pixelTrim?: number): Promise { - const startTime = performance.now(); - const item = await fsa.readJson(input); - const jobId = ulid.ulid(); - const tmpFolder = new URL(`${jobId}/`, tmp); - try { - // Extract the source URL from the item - logger.info({ item: item.id }, 'CogCreation:Start'); - const source = item.assets.source.href; - await mkdir(tmpFolder, { recursive: true }); - - // Download the source file - const sourceUrl = tryParseUrl(source); - if (!(await fsa.exists(sourceUrl))) throw new Error('Source file not found'); - const hashStreamSource = fsa.readStream(sourceUrl).pipe(new HashTransform('sha256')); - - const filePath = path.parse(sourceUrl.href); - const fileName = filePath.base; - const inputPath = new URL(fileName, tmpFolder); - - logger.info({ item: item.id, download: inputPath.href }, 'CogCreation:Download'); - // Add checksum for source file - if (item.assets['source'] == null) throw new Error('No source file found in the item'); - await fsa.write(inputPath, hashStreamSource); - item.assets['source']['file:size'] = hashStreamSource.size; - item.assets['source']['file:checksum'] = hashStreamSource.multihash; - - // read resolution from first image of tiff - const tiff = await new Tiff(fsa.source(inputPath)).init(); - const sourceRes = tiff.images[0]?.resolution; - if (sourceRes == null) throw new Error('Could not read resolution from first image'); - - // run gdal commands for each source file - - /** - * command: gdal build vrt - */ - logger.info({ item: item.id }, 'CogCreation:gdalbuildvrt'); - - const vrtPath = new URL(`${item.id}.vrt`, tmpFolder); - const commandBuildVrt = gdalBuildVrt(vrtPath, [inputPath]); - - await new GdalRunner(commandBuildVrt).run(logger); - - /** - * command: gdal build cog - */ - logger.info({ item: item.id }, 'CogCreation:gdal_translate'); - - const width = item.properties['source.width']; - const height = item.properties['source.height']; - const tempPath = new URL(`${item.id}.tiff`, tmpFolder); - const commandTranslate = gdalBuildCog(vrtPath, tempPath, width, height, { pixelTrim }); - - await new GdalRunner(commandTranslate).run(logger); - - // fsa.write output to target location - logger.info({ item: item.id }, 'CogCreation:Output'); - const readStream = fsa.readStream(tempPath).pipe(new HashTransform('sha256')); - const outputPath = tryParseUrl(input.href.replace('.json', '.tiff')); - await fsa.write(outputPath, readStream); - - // Add the asset of created cog into stac item - const stac = await fsa.read(outputPath); - item.assets.cog = { - href: `./${item.id}.tiff`, - type: 'image/tiff; application=geotiff; profile=cloud-optimized', - roles: ['data'], - ...createFileStats(stac), - }; - await fsa.write(input, JSON.stringify(item, null, 2)); - } finally { - // Cleanup the temporary folder once everything is done - logger.info({ path: tmpFolder.href }, 'CogCreation:Cleanup'); - await rm(tmpFolder.href, { recursive: true, force: true }); - logger.info({ duration: performance.now() - startTime }, 'CogCreation:Done'); - } -} diff --git a/src/commands/index.ts b/src/commands/index.ts index 83998879..9304dbae 100644 --- a/src/commands/index.ts +++ b/src/commands/index.ts @@ -3,7 +3,6 @@ import { subcommands } from 'cmd-ts'; import { CliInfo } from '../cli.info.js'; import { basemapsCreatePullRequest } from './basemaps-github/create-pr.js'; import { basemapsCreateMapSheet } from './basemaps-mapsheet/create-mapsheet.js'; -import { topoCogCreation } from './basemaps-topo-import/topo-cog-creation.js'; import { topoStacCreation } from './basemaps-topo-import/topo-stac-creation.js'; import { commandCopy } from './copy/copy.js'; import { commandCreateManifest } from './create-manifest/create-manifest.js'; @@ -51,7 +50,6 @@ export const AllCommands = { 'create-pr': basemapsCreatePullRequest, 'create-mapsheet': basemapsCreateMapSheet, 'topo-stac-creation': topoStacCreation, - 'topo-cog-creation': topoCogCreation, }, }), 'pretty-print': commandPrettyPrint, From 6f1f8b0c9ddf2d921df7b8b135a88c690d67d949 Mon Sep 17 00:00:00 2001 From: Wentao Kuang Date: Mon, 13 Jan 2025 10:22:53 +1300 Subject: [PATCH 47/47] Remove docker in the container. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a1164701..1ce55934 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM node:20-slim@sha256:cffed8cd39d6a380434e6d08116d188c53e70611175cd5ec7700f93f32a935a6 -RUN apt-get update && apt-get install openssh-client git docker.io -y +RUN apt-get update && apt-get install openssh-client git -y WORKDIR /app