feat(pull lexicons) sort lexicon slots and synonyms alphabetically

src/lib/endpoints.ts

@@ -8,6 +8,7 @@ import CONFIG from '../utils/config';
 import CognigyClient from '../utils/cognigyClient';
 import { checkCreateDir, removeCreateDir } from '../utils/checks';
 import { indexAll } from '../utils/indexAll';
+import { sortUtils } from '../utils/sortUtils';
 
 
 /**
@@ -40,7 +41,7 @@ export const cloneEndpoints = async (availableProgress: number): Promise<void> =
         });
 
         await removeCreateDir(individualEndpointDir);
-        fs.writeFileSync(individualEndpointDir + "/config.json", JSON.stringify(endpointDetail, undefined, 4));
+        fs.writeFileSync(individualEndpointDir + "/config.json", JSON.stringify(sortUtils.sortObj(endpointDetail), undefined, 4));
         fs.writeFileSync(individualEndpointDir + "/transformer.ts", endpointDetail.transformer.transformer);
         addToProgressBar(incrementPerEndpoint);
     }
@@ -90,7 +91,7 @@ export const pullEndpoint = async (endpointName: string, availableProgress: numb
     });
 
     // write files to disk
-    fs.writeFileSync(endpointDir + "/config.json", JSON.stringify(endpointDetail, undefined, 4));
+    fs.writeFileSync(endpointDir + "/config.json", JSON.stringify(sortUtils.sortObj(endpointDetail), undefined, 4));
     fs.writeFileSync(endpointDir + "/transformer.ts", endpointDetail.transformer.transformer);
     addToProgressBar(70);
 

src/lib/flows.ts

@@ -17,6 +17,7 @@ import translateFlowNode, { translateIntentExampleSentence, translateSayNode } f
 import { makeAxiosRequest } from '../utils/axiosClient';
 
 import { indexAll } from '../utils/indexAll';
+import { sortUtils } from '../utils/sortUtils';
 
 // Interfaces
 import { ILocaleIndexItem_2_0 } from '@cognigy/rest-api-client/build/shared/interfaces/restAPI/resources/locales/v2.0';
@@ -84,7 +85,7 @@ export const pullFlow = async (flowName: string, availableProgress: number): Pro
 
         await removeCreateDir(localeDir);
 
-        fs.writeFileSync(flowDir + "/config.json", JSON.stringify(flow, undefined, 4));
+        fs.writeFileSync(flowDir + "/config.json", JSON.stringify(sortUtils.sortObj(flow), undefined, 4));
 
         const chart = await CognigyClient.readChart({
             "resourceId": flow._id,
@@ -117,7 +118,20 @@ export const pullFlow = async (flowName: string, availableProgress: number): Pro
             type: null
         });
 
-        fs.writeFileSync(localeDir + "/intents.json", JSON.stringify(flowIntents, undefined, 4));
+        let sortedFlowIntents = [];
+        flowIntents.forEach(flowIntent => {
+            flowIntent = sortUtils.sortObj(flowIntent);
+            if (flowIntent.confirmationSentences.length > 1) {
+                flowIntent.confirmationSentences.sort(sortUtils.sortI);
+            }
+            if (flowIntent.exampleSentences.length > 1) {
+                flowIntent.exampleSentences.sort(sortUtils.sortI);
+            }
+            sortedFlowIntents.push(flowIntent);
+        });
+
+        sortedFlowIntents.sort(sortUtils.sortIByNameKey);
+        fs.writeFileSync(localeDir + "/intents.json", JSON.stringify(sortedFlowIntents, undefined, 4));
     }
 
     return Promise.resolve();

src/lib/lexicons.ts

@@ -11,6 +11,7 @@ import CognigyClient from '../utils/cognigyClient';
 import { makeAxiosRequest } from '../utils/axiosClient';
 import { checkCreateDir, checkTask, removeCreateDir } from '../utils/checks';
 import { indexAll } from '../utils/indexAll';
+import { sortUtils } from '../utils/sortUtils';
 
 /**
  * Clones Cognigy Lexicons to disk
@@ -86,17 +87,17 @@ export const pullLexicon = async (lexiconName: string, availableProgress: number
         lexiconId: lexicon._id
     };
 
-    fs.writeFileSync(lexiconDir + "/config.json", JSON.stringify(lexiconConfig, undefined, 4));
+    fs.writeFileSync(lexiconDir + "/config.json", JSON.stringify(sortUtils.sortObj(lexiconConfig), undefined, 4));
 
     // pull lexicon data from Cognigy.AI
     const csvData = await CognigyClient.exportFromLexicon({
         lexiconId: lexicon._id,
         projectId: CONFIG.agent,
         type: 'text/csv'
     });
-
+    const sortedCsvData = sortUtils.sortLexiconCsv(csvData);
     // write files to disk
-    fs.writeFileSync(lexiconDir + "/keyphrases.csv", csvData);
+    fs.writeFileSync(lexiconDir + "/keyphrases.csv", sortedCsvData);
     addToProgressBar(70);
 
     return Promise.resolve();

src/utils/sortUtils.ts

@@ -0,0 +1,78 @@
+export const sortUtils = {
+	sortObj: (unordered, sortArrays = false) => {
+		if (!unordered || typeof unordered !== 'object') {
+			return unordered;
+		}
+
+		if (Array.isArray(unordered)) {
+			const newArr = unordered.map((item) => sortUtils.sortObj(item, sortArrays));
+			if (sortArrays) {
+				newArr.sort(sortUtils.sortI);
+			}
+			return newArr;
+		}
+
+		const ordered = {};
+		Object.keys(unordered)
+			.sort(sortUtils.sortI)
+			.forEach((key) => {
+				ordered[key] = sortUtils.sortObj(unordered[key], sortArrays);
+			});
+		return ordered;
+	},
+
+	sortI: (a, b) => {
+		var nameA = a.toUpperCase(); // ignore upper and lowercase
+		var nameB = b.toUpperCase(); // ignore upper and lowercase
+		if (nameA < nameB) {
+			return -1;
+		}
+		if (nameA > nameB) {
+			return 1;
+		}
+
+		// names must be equal
+		return 0;
+	},
+	sortIByNameKey: (a, b) => {
+		var nameA = a.name.toUpperCase(); // ignore upper and lowercase
+		var nameB = b.name.toUpperCase(); // ignore upper and lowercase
+		if (nameA < nameB) {
+			return -1;
+		}
+		if (nameA > nameB) {
+			return 1;
+		}
+
+		// names must be equal
+		return 0;
+	},
+	sortLexiconCsv: (csvData: string) => {
+		let csvArr = csvData.split("\n");
+
+		csvArr = csvArr.sort(sortUtils.sortI);
+		let sortedCsvArr = [];
+
+		csvArr.forEach(line => {
+			const splitByNonNestedDoubleQuotes = /[,](?=(?:[^"]|"[^"]*")*$)/g;
+			const lineArr = line.split(splitByNonNestedDoubleQuotes);
+
+			// remove leading and trailing " for each csv column
+			const keyphrase = lineArr[0].slice(1, -1);
+			const slots = lineArr[1].slice(1, -1);
+			const synonyms = lineArr[2].slice(1, -1);
+			const slotsArr = sortUtils.sortStringByNonNestedComma(slots);
+			const synonymsArr = sortUtils.sortStringByNonNestedComma(synonyms);
+
+			sortedCsvArr.push('"' + keyphrase + '","' + slotsArr.join(',') + '","' + synonymsArr.join(',') + '",');
+		});
+
+		return sortedCsvArr.join("\n");
+	},
+	sortStringByNonNestedComma: (str: string) => {
+		const splitByNonNestedSingleQuotes = /[,](?=(?:[^']|'[^']*')*$)/g;
+		const strList = str.split(splitByNonNestedSingleQuotes);
+		let sortedStrList = strList.sort(sortUtils.sortI);
+		return sortedStrList;
+	}
+};