Add ttl to fetch promises, check metadata rows before fetching

README.md

@@ -119,3 +119,36 @@ const levels = {
    - Click on the Import button.
 
 9. Open the browser and navigate to http://localhost:8001/mosaic-viewer to view the mosaic map.
+
+## Environment variables
+
+| Name                    | Description                             | Default Value   |
+| ----------------------- | --------------------------------------- | --------------- |
+| `PGHOST`                | PostgreSQL host                         |                 |
+| `PGUSER`                | PostgreSQL user                         |                 |
+| `PGPASSWORD`            | PostgreSQL password                     |                 |
+| `PGDATABASE`            | PostgreSQL database name                | `postgres`      |
+| `OAM_LAYER_ID`          | OpenAerialMap layer ID                  | `openaerialmap` |
+| `PORT`                  | Server port                             |                 |
+| `BASE_URL`              | Root URL of the server                  |                 |
+| `TITILER_BASE_URL`      | URL of your Titiler installation        |                 |
+| `TILES_CACHE_DIR_PATH`  | Path for the tiles cache                | `/tiles`        |
+| `LOG_LEVEL`             | Logging level                           |                 |
+| `DB_POOL_SIZE`          | Size of the PostgreSQL connection pool  | `16`            |
+| `DB_DISABLE_SSL`        | Disable SSL for PostgreSQL connection   | `false`         |
+| `TILE_FETCH_TIMEOUT_MS` | Maximum time to wait for tile fetch (ms). Requests exceeding this timeout will fail with a timeout error. | `60000`         |
+| `FETCH_QUEUE_TTL_MS`    | Time to keep fetch promises in memory (ms). Older promises are removed to prevent memory leaks. | `600000`        |
+
+## Tests
+
+To run tests:
+
+```bash
+npm test
+```
+
+To update snapshots:
+
+```bash
+UPDATE_SNAPSHOTS=1 npm test
+```

__tests__/mosaic.mjs

@@ -176,6 +176,12 @@ test.skip("mosaic(14, 9485, 5610) and 2 parent tiles", async () => {
     requestCachedMosaic512px(12, 2371, 1402),
   ]);
 
+  if (process.env.UPDATE_SNAPSHOTS) {
+    fs.writeFileSync("./__tests__/[email protected]", tile.image.buffer);
+    fs.writeFileSync("./__tests__/[email protected]", parentTile.image.buffer);
+    fs.writeFileSync("./__tests__/[email protected]", parentParentTile.image.buffer);
+  }
+
   expect(
     compareTilesPixelmatch(
       fs.readFileSync("./__tests__/[email protected]"),
@@ -229,6 +235,10 @@ test("mosaic256px(14, 9485, 5610)", async () => {
 
   const tile = await requestCachedMosaic256px(15, 18970, 11220);
 
+  if (process.env.UPDATE_SNAPSHOTS) {
+    fs.writeFileSync("./__tests__/[email protected]", tile.image.buffer);
+  }
+
   expect(
     compareTilesPixelmatch(
       fs.readFileSync("./__tests__/[email protected]"),
@@ -262,6 +272,11 @@ test("mosaic(11, 1233, 637)", async () => {
   expect(metadataRequestQueue.size).toBe(0);
 
   const expected = fs.readFileSync("./__tests__/[email protected]");
+
+  if (process.env.UPDATE_SNAPSHOTS) {
+    fs.writeFileSync("./__tests__/[email protected]", tile.image.buffer);
+  }
+
   expect(compareTilesPixelmatch(expected, tile.image.buffer, 512)).toBe(0);
 });
 

src/mosaic.mjs

@@ -171,10 +171,11 @@ async function mosaic512px(z, x, y, filters = {}) {
   const metadataByUuid = {};
   await Promise.all(
     rows.map(async (row) => {
-      metadataByUuid[row.uuid] = await getGeotiffMetadata(row.uuid);
+      if (row?.uuid) {
+        metadataByUuid[row.uuid] = await getGeotiffMetadata(row.uuid);
+      }
     })
   );
-
   const tilePromises = [];
   if (z < 9) {
     for (const row of rows) {
@@ -224,7 +225,14 @@ async function mosaic512px(z, x, y, filters = {}) {
     .map((tile, index) => ({
       tile,
       meta: metadataByUuid[rows[index].uuid],
-    }));
+    }))
+    .filter(({ meta, tile }, index) => {
+      if (!meta) {
+        console.warn(`Null metadata found for tile at index ${index}, skipping...`);
+        return false;
+      }
+      return true;
+    });
 
   // Sort tiles based on the criteria
   filteredTiles.sort((a, b) => {

src/mosaic_viewer.ejs

@@ -38,7 +38,7 @@
     <script>
       // get zoom and center from url hash
       const hash = window.location.hash.slice(1).split("/");
-      const [zoom = 0, lng = 0, lat = 0] = hash.filter(Boolean).map(parseFloat);
+      const [zoom = 0, lat = 0, lng = 0] = hash.filter(Boolean).map(parseFloat);
       mapboxgl.accessToken = "";
       const map = new mapboxgl.Map({
         container: "map",
@@ -96,7 +96,7 @@
       map.on("moveend", () => {
         const { lng, lat } = map.getCenter();
         const zoom = map.getZoom();
-        window.location.hash = `#${zoom.toFixed(2)}/${lng.toFixed(4)}/${lat.toFixed(4)}`;
+        window.location.hash = `#${zoom.toFixed(2)}/${lat.toFixed(4)}/${lng.toFixed(4)}`;
       });
     </script>
   </body>

src/titiler_fetcher.mjs

@@ -12,9 +12,13 @@ const TITILER_BASE_URL = process.env.TITILER_BASE_URL;
 const tileRequestQueue = new PQueue({ concurrency: numCPUs });
 const activeTileRequests = new Map();
 
+const TILE_FETCH_TIMEOUT_MS =
+  Number.parseInt(process.env.TILE_FETCH_TIMEOUT_MS, 10) || 1000 * 60 * 1; // 1 minute default
+
 async function fetchTile(url) {
   try {
     const responsePromise = got(url, {
+      timeout: { request: TILE_FETCH_TIMEOUT_MS },
       throwHttpErrors: true,
     });
 
@@ -34,14 +38,30 @@ async function fetchTile(url) {
   }
 }
 
+const FETCH_QUEUE_TTL_MS = Number.parseInt(process.env.FETCH_QUEUE_TTL_MS, 10) || 1000 * 60 * 10; // 10 minutes default
+
 async function enqueueTileFetching(tileUrl, z, x, y) {
   const url = tileUrl.replace("{z}", z).replace("{x}", x).replace("{y}", y);
   if (activeTileRequests.get(url)) {
     return activeTileRequests.get(url);
   }
 
   const request = tileRequestQueue
-    .add(() => fetchTile(url), { priority: z })
+    .add(() => fetchTile(url), { priority: Math.pow(2, z), timeout: FETCH_QUEUE_TTL_MS })
+    .catch((error) => {
+      const logContext = {
+        url,
+        zoomLevel: z,
+        errorType: error.name,
+        errorMessage: error.message,
+        timeout: FETCH_QUEUE_TTL_MS
+      };
+      if (error.name === "TimeoutError") {
+        console.error('Tile request timeout', logContext);
+      } else {
+        console.error('Tile request failed', logContext);
+      }
+    })
     .finally(() => {
       activeTileRequests.delete(url);
     });
@@ -62,7 +82,7 @@ async function fetchTileMetadata(uuid) {
     const metadata = await got(url.href).json();
     return metadata;
   } catch (err) {
-    if (err.response && (err.response.statusCode === 404 || err.response.statusCode === 500)) {
+    if ([404, 500].includes(err?.response?.statusCode)) {
       return null;
     } else {
       throw err;