Merge branch 'main' into unified-ontology

Modules/IntentSpecification2WorkflowGenerator/api/api_main.py

@@ -63,6 +63,7 @@ def convert_strings_to_uris(obj):
 @app.post('/logical_planner')
 def run_logical_planner():
     plan_ids = request.json.get('plan_ids', '')
+    print(plan_ids)
     intent_json = request.json.get('intent_graph', '')
     algorithm_implementations = request.json.get('algorithm_implementations', '')
     ontology = Graph().parse(data=request.json.get('ontology', ''), format='turtle')
@@ -71,14 +72,15 @@ def run_logical_planner():
     algorithm_implementations_uris = convert_strings_to_uris(algorithm_implementations)
 
     intent = Graph().parse(data=intent_json, format='turtle')
-    intent.print()
 
     impls = [impl
              for alg, impls in algorithm_implementations_uris.items() if str(alg) in plan_ids
              for impl in impls]
 
     workflow_plans = workflow_planner(ontology, impls, intent)
+    workflow_plans[0].serialize(destination='C:\\Users\\marc.maynou\\Desktop\\NextiaJD\\wplan.rdf', format='xml')
     logical_plans = logical_planner(ontology, workflow_plans)
+    workflow_plans[0].serialize(destination='C:\\Users\\marc.maynou\\Desktop\\NextiaJD\\lplan.rdf', format='xml')
 
     return logical_plans
 
@@ -203,38 +205,32 @@ def download_proactive():
         load_dataset_task = bucket.create_Import_Data_task(import_from="PA:USER_FILE", file_path=data_product_name + ".csv", file_delimiter=";", label_column=label_column)
         proactive_job.addTask(load_dataset_task)
 
+        # remove_nulls = bucket.create_Fill_NaNs_task(0)
+        # split_data_task.addDependency(load_dataset_task)
+        # proactive_job.addTask(remove_nulls)
+
         split_data_task = bucket.create_Split_Data_task()
         split_data_task.addDependency(load_dataset_task)
         proactive_job.addTask(split_data_task)
 
         # Model depends on the layout, the rest is the same
-        scale_task = bucket.create_Scale_Data_task()
         model_task = bucket.create_Support_Vector_Machines_task()
         for key in layout:
             if "decision_tree_predictor" in key:
                 model_task = bucket.create_Random_Forest_task()
                 break
-        random_forest_task = bucket.create_Random_Forest_task()
-        proactive_job.addTask(random_forest_task)
+        proactive_job.addTask(model_task)
 
         train_model_task = bucket.create_Train_Model_task()
         train_model_task.addDependency(split_data_task)
         train_model_task.addDependency(model_task)
         proactive_job.addTask(train_model_task)
 
-        download_model_task = bucket.create_Download_Model_task()
-        download_model_task.addDependency(train_model_task)
-        proactive_job.addTask(download_model_task)
-
         predict_model_task = bucket.create_Predict_Model_task()
         predict_model_task.addDependency(split_data_task)
         predict_model_task.addDependency(train_model_task)
         proactive_job.addTask(predict_model_task)
 
-        preview_results_task = bucket.create_Preview_Results_task()
-        preview_results_task.addDependency(predict_model_task)
-        proactive_job.addTask(preview_results_task)
-
         gateway.saveJob2XML(proactive_job, os.path.abspath(r'api/temp_files/extremexp_test_workflow.xml'))
 
     finally:

Modules/NextiaBS/src/main/java/edu/upc/essi/dtim/nextiabs/implementations/CSVBootstrap.java

@@ -12,13 +12,9 @@
 import edu.upc.essi.dtim.nextiabs.bootstrap.BootstrapResult;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
 
-import java.io.BufferedReader;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Objects;
+import java.io.*;
 import java.util.stream.Collectors;
 
 import static edu.upc.essi.dtim.nextiabs.utils.DF_MMtoRDFS.productionRulesDataframe_to_RDFS;
@@ -44,17 +40,13 @@ public Graph bootstrapSchema(Boolean generateMetadata) {
 		G_target = (LocalGraph) CoreGraphFactory.createGraphInstance("local");
 //		setPrefixes();
 
-		BufferedReader br;
-		try {
-			br = new BufferedReader(new FileReader(path));
-		} catch (FileNotFoundException e) {
-			throw new RuntimeException(e);
-		}
-
 		CSVParser parser;
 		try {
-			parser = CSVParser.parse(br, CSVFormat.DEFAULT.withFirstRecordAsHeader());
+			char delimiter = detectDelimiter(path); // Detect the delimiter of the file
+			BufferedReader br = new BufferedReader(new FileReader(path));
+			parser = CSVParser.parse(br, CSVFormat.DEFAULT.withFirstRecordAsHeader().withDelimiter(delimiter));
 		} catch (IOException e) {
+			e.printStackTrace();
 			throw new RuntimeException(e);
 		}
 
@@ -68,7 +60,7 @@ public Graph bootstrapSchema(Boolean generateMetadata) {
 			G_target.addTriple(createIRI(h2),DataFrame_MM.hasDataType,DataFrame_MM.String);
 		});
 
-		String select =  parser.getHeaderNames().stream().map(a ->  a + " AS " + reformatName(a)).collect(Collectors.joining(", "));
+		String select =  parser.getHeaderNames().stream().map(a ->  "\"" + a + "\" AS " + reformatName(a)).collect(Collectors.joining(", "));
 		wrapper = "SELECT " + select  + " FROM `" + name + "`";
 
 		//TODO: implement metadata
@@ -95,6 +87,30 @@ public void generateMetadata(){
 		G_target.addTripleLiteral(ds, DataSourceVocabulary.HAS_WRAPPER.getURI(), wrapper);
 	}
 
+	private char detectDelimiter(String path) throws IOException {
+		char[] delimiters = {';', ',', '\t'};
+		BufferedReader br = new BufferedReader(new FileReader(path));
+
+		for (char delimiter : delimiters) {
+			// Parsing the CSV file with current delimiter
+			CSVFormat csvFormat = CSVFormat.DEFAULT.withDelimiter(delimiter);
+			CSVParser csvParser = new CSVParser(br, csvFormat);
+
+			Iterable<CSVRecord> records = csvParser.getRecords(); // Get the first record
+			if (records.iterator().hasNext()) {
+				CSVRecord firstRecord = records.iterator().next();
+				// If the record contains more than 1 column, we assume it's the correct delimiter
+				if (firstRecord.size() > 1) {
+					csvParser.close();
+					return delimiter;
+				}
+			}
+			csvParser.close(); // Close the parser
+			br = new BufferedReader(new FileReader(path)); // Reset the reader to start from the beginning of the file
+		}
+		return ','; // Return null if no delimiter is detected
+	}
+
 	@Override
 	public Graph bootstrapSchema() {
 		return bootstrapSchema(false);

Modules/NextiaJD/build.gradle

@@ -1,5 +1,6 @@
 plugins {
     id 'java'
+    id 'com.github.johnrengelman.shadow' version '7.1.2'
 }
 
 group = 'edu.upc.essi.dtim'
@@ -22,11 +23,20 @@ dependencies {
     implementation group: 'com.googlecode.json-simple', name: 'json-simple', version: '1.1.1'
     implementation group: 'org.jpmml', name: 'pmml-evaluator', version: '1.6.4'
     implementation group: 'commons-io', name: 'commons-io', version: '2.15.0'
+    implementation group: 'com.opencsv', name: 'opencsv', version: '5.9'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
 }
 
+shadowJar {
+    zip64(true)
+    manifest {
+        attributes 'Main-Class': 'edu.upc.essi.dtim.NextiaJD.Main'
+    }
+}
+
 test {
     useJUnitPlatform()
-}
+}
+