Add an API endpoint and UI to "add" a table to config (#75)

api/dataProcessing/filterData.ts

@@ -1,4 +1,4 @@
-import { Column } from "./types";
+import { Column } from "../types";
 import { hasValidCoordinates } from "./helpers";
 
 // Filter out unwanted columns and substrings

api/dataProcessing/transformData.ts

@@ -1,4 +1,4 @@
-import { AlertRecord, Metadata, AlertsPerMonth } from "./types";
+import { AlertRecord, Metadata, AlertsPerMonth } from "../types";
 import {
   capitalizeFirstLetter,
   getRandomColor,

api/database/dbOperations.ts

@@ -1,3 +1,5 @@
+import { Views } from "../types";
+
 const checkTableExists = (
   db: any,
   table: string | undefined,
@@ -96,33 +98,28 @@ export const fetchData = async (
   return { mainData, columnsData, metadata };
 };
 
-interface ViewConfig {
-  VIEWS: string;
-  FILTER_BY_COLUMN: string;
-  FILTER_OUT_VALUES_FROM_COLUMN: string;
-  FRONT_END_FILTER_COLUMN: string;
-  MAPBOX_STYLE: string;
-  MAPBOX_PROJECTION: string;
-  MAPBOX_CENTER_LATITUDE: string;
-  MAPBOX_CENTER_LONGITUDE: string;
-  MAPBOX_ZOOM: string;
-  MAPBOX_PITCH: string;
-  MAPBOX_BEARING: string;
-  MAPBOX_3D: string;
-  MAPEO_TABLE: string;
-  MAPEO_CATEGORY_IDS: string;
-  MAP_LEGEND_LAYER_IDS: string;
-  MEDIA_BASE_PATH: string;
-  MEDIA_BASE_PATH_ALERTS: string;
-  LOGO_URL: string;
-  PLANET_API_KEY: string;
-  UNWANTED_COLUMNS?: string;
-  UNWANTED_SUBSTRINGS?: string;
-}
-
-interface Views {
-  [key: string]: ViewConfig;
-}
+export const fetchTableNames = async (
+  db: any,
+  isSQLite: boolean | undefined,
+): Promise<string[]> => {
+  const query = isSQLite
+    ? `SELECT name FROM sqlite_master WHERE type='table'`
+    : `SELECT table_name FROM information_schema.tables WHERE table_schema = 'public'`;
+
+  return new Promise((resolve, reject) => {
+    if (isSQLite) {
+      db.all(query, (err: Error, rows: any[]) => {
+        if (err) reject(err);
+        resolve(rows.map((row) => row.name));
+      });
+    } else {
+      db.query(query, (err: Error, result: { rows: any[] }) => {
+        if (err) reject(err);
+        resolve(result.rows.map((row) => row.table_name));
+      });
+    }
+  });
+};
 
 export const fetchConfig = async (
   db: any,
@@ -208,6 +205,38 @@ export const updateConfig = async (
   });
 };
 
+export const addNewTableToConfig = async (
+  db: any,
+  tableName: string,
+  isSQLite: boolean | undefined,
+): Promise<void> => {
+  const query = isSQLite
+    ? `INSERT INTO config (table_name, views_config) VALUES (?, ?)`
+    : `INSERT INTO config (table_name, views_config) VALUES ($1, $2)`;
+
+  return new Promise((resolve, reject) => {
+    if (isSQLite) {
+      db.run(query, [tableName, "{}"], (err: Error) => {
+        if (err) {
+          console.error("SQLite Error:", err);
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    } else {
+      db.query(query, [tableName, "{}"], (err: Error) => {
+        if (err) {
+          console.error("PostgreSQL Error:", err);
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    }
+  });
+};
+
 export const removeTableFromConfig = async (
   db: any,
   tableName: string,

api/index.ts

@@ -6,7 +6,9 @@ import {
 } from "./database/dbConnection";
 import {
   fetchData,
+  fetchTableNames,
   fetchConfig,
+  addNewTableToConfig,
   updateConfig,
   removeTableFromConfig,
 } from "./database/dbOperations";
@@ -36,6 +38,7 @@ import {
   IS_SQLITE,
   SQLITE_DB_PATH,
 } from "./config";
+import { Views } from "./types";
 
 let configDb: any;
 let db: any;
@@ -50,14 +53,27 @@ app.post("/login", postLogin);
 // Apply middleware to views routes
 app.use(checkAuthStrategy);
 
-let viewsConfig: any = {};
+let viewsConfig: Views = {};
 
 // Fetch views config
 const getViewsConfig = async () => {
   viewsConfig = await fetchConfig(configDb, IS_SQLITE);
   return viewsConfig;
 };
 
+// Fetch table names
+const getTableNames = async () => {
+  let tableNames = await fetchTableNames(db, IS_SQLITE);
+  // Filter out anything with metadata, columns, and anything PostGIS related
+  tableNames = tableNames.filter(
+    (name) =>
+      !name.includes("metadata") &&
+      !name.includes("columns") &&
+      !name.includes("spatial_ref_sys"),
+  );
+  return tableNames;
+};
+
 // Initialize views using config
 const initializeViewsConfig = async () => {
   // Define allowed file extensions
@@ -77,7 +93,7 @@ const initializeViewsConfig = async () => {
   tableNames.forEach((table) => {
     // Check if viewsConfig[table].viewsConfig is not set
     if (!viewsConfig[table].VIEWS) {
-      console.log(`viewsConfig not defined for ${table}, skipping...`);
+      console.warn(`viewsConfig not defined for ${table}, skipping...`);
       return;
     }
 
@@ -376,7 +392,14 @@ const setupAndInitialize = async () => {
 // GET views configuration
 app.get("/config", async (_req: express.Request, res: express.Response) => {
   try {
-    res.json(await getViewsConfig());
+    let tableNames = await getTableNames();
+
+    // Filter out any tables that are already in viewsConfig
+    tableNames = tableNames.filter(
+      (name) => !Object.keys(viewsConfig).includes(name),
+    );
+
+    res.json([viewsConfig, tableNames]);
   } catch (error: any) {
     console.error("Error fetching views configuration:", error.message);
     res.status(500).json({ error: error.message });
@@ -406,6 +429,28 @@ app.post(
   },
 );
 
+// POST a new table to config
+app.post(
+  "/config/new-table/:tableName",
+  async (req: express.Request, res: express.Response) => {
+    const { tableName } = req.params;
+
+    try {
+      await addNewTableToConfig(configDb, tableName, IS_SQLITE);
+      res.json({ message: "New table added successfully" });
+
+      // Reinitialize viewsConfig with updated config
+      await getViewsConfig();
+      initializeViewsConfig().catch((error) => {
+        console.error("Error reinitializing views config:", error.message);
+      });
+    } catch (error: any) {
+      console.error("Error updating config:", error.message);
+      res.status(500).json({ error: error.message });
+    }
+  },
+);
+
 // DELETE a table record from configuration
 app.delete(
   "/config/:tableName",

api/types.ts

@@ -0,0 +1,52 @@
+export type Column = {
+  original_column: string;
+  sql_column: string;
+};
+
+export type AlertRecord = {
+  territory_name: string;
+  alert_type: string;
+  month_detec: string;
+  year_detec: string;
+  area_alert_ha: string;
+  _topic: string;
+};
+
+export type Metadata = {
+  type_alert: string;
+  month: number;
+  year: number;
+  total_alerts: string;
+  description_alerts: string;
+};
+
+export type AlertsPerMonth = Record<string, number>;
+
+export interface ViewConfig {
+  VIEWS: string;
+  FILTER_BY_COLUMN: string;
+  FILTER_OUT_VALUES_FROM_COLUMN: string;
+  FRONT_END_FILTER_COLUMN: string;
+  MAPBOX_ACCESS_TOKEN: string;
+  MAPBOX_STYLE: string;
+  MAPBOX_PROJECTION: string;
+  MAPBOX_CENTER_LATITUDE: string;
+  MAPBOX_CENTER_LONGITUDE: string;
+  MAPBOX_ZOOM: string;
+  MAPBOX_PITCH: string;
+  MAPBOX_BEARING: string;
+  MAPBOX_3D: string;
+  MAPEO_TABLE: string;
+  MAPEO_CATEGORY_IDS: string;
+  MAP_LEGEND_LAYER_IDS: string;
+  MEDIA_BASE_PATH: string;
+  MEDIA_BASE_PATH_ALERTS: string;
+  LOGO_URL: string;
+  PLANET_API_KEY: string;
+  UNWANTED_COLUMNS?: string;
+  UNWANTED_SUBSTRINGS?: string;
+}
+
+export interface Views {
+  [key: string]: ViewConfig;
+}