Merge pull request #446 from WildMeOrg/bulk-import-validation

Bulk import validation - species

src/main/java/org/ecocean/importutils/RowFeedback.java

@@ -58,20 +58,31 @@ public void logParseValue(int colNum, Object value, Row row) {
         } 
       }
     }
-
+    
     public void logParseError(int colNum, Object value, Row row) {
+    	logParseError(colNum, value, row, null);
+    }
+
+    public void logParseError(int colNum, Object value, Row row, String exactMessageToDisplay) {
       try {
         if (!committing) {
           if (colNum<this.cells.length&&this.cells[colNum]!=null) {
             CellFeedback cellFeedback = this.cells[colNum];
             System.out.println("Setting ERROR value on OLD CellFeedback for col "+colNum+" val "+String.valueOf(value)+" row "+row.getRowNum());
             // I think we can assume a BLANK or NULL cell doesn't need to get overwritten with an error. 
-            if (!cellFeedback.isBlank()) {
-              this.cells[colNum].setSuccess(false);
-              //TODO replace this universal NOT FOUND for an overwrite with something specific.
-              this.cells[colNum].setValueString(value+" NOT FOUND");
+            //if (!cellFeedback.isBlank()) {
+            this.cells[colNum].setSuccess(false);
+            this.cells[colNum].setIsBlank(false);
+            //TODO replace this universal NOT FOUND for an overwrite with something specific.
+            if(exactMessageToDisplay!=null) {
+            	this.cells[colNum].setValueString(exactMessageToDisplay);
             }
-          } else {
+            else {
+            	this.cells[colNum].setValueString(value+" NOT FOUND");
+            }
+            //}
+          } 
+          else {
             System.out.println("Setting ERROR value on NEW CellFeedback for col "+colNum+" val "+String.valueOf(value)+" row "+row.getRowNum());
             this.cells[colNum] = new CellFeedback(value, false, false);
           }

src/main/java/org/ecocean/importutils/TabularFeedback.java

@@ -100,10 +100,14 @@ public void logParseValue(int colNum, Object value, Row row) {
     }
 
     public void logParseError(int colNum, Object value, Row row) {
-      if (!committing) {
-        this.currentRow.logParseError(colNum, value, row);
-      }
+    	logParseError(colNum, value, row, null);
     }
+
+    public void logParseError(int colNum, Object value, Row row, String exactMessageToDisplay) {
+        if (!committing) {
+          this.currentRow.logParseError(colNum, value, row, exactMessageToDisplay);
+        }
+      }
 
     public void logParseNoValue(int colNum) {
       if (!committing) {

src/main/java/org/ecocean/servlet/importer/StandardImport.java

@@ -44,6 +44,8 @@
 import org.json.JSONArray;
 import org.json.JSONObject;
 
+
+
 import javax.servlet.RequestDispatcher;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletContext;
@@ -551,7 +553,7 @@ public void doImport(String filename, File dataFile, HttpServletRequest request,
     }
 
     out.println("<div class=\"col-sm-12 col-md-6 col-lg-6 col-xl-6\">"); // half page bootstrap column
-    out.println("<h2>Import Overview: </h2>");
+    out.println("<h2>Import Overview</h2>");
     out.println("<ul>");
     out.println("<li>Excel File Name: "+filename+"</li>");
     out.println("<li>Excel File Successfully Found = "+dataFound+"</li>");
@@ -560,8 +562,43 @@ public void doImport(String filename, File dataFile, HttpServletRequest request,
     out.println("<li>Excel Columns = "+cols+"</li>");
     //out.println("<li>Last col num = "+lastColNum+"</li>");
     out.println("<li><em>Trial Run: "+!committing+"</em></li>");
-
     out.println("</ul>");
+
+    out.println("<h3 id='errorHeader' style='color: red;'>Errors</h3>");
+    out.println("<p>Number of errors: <span id='errorCount'></span></p>");
+    out.println("<p>Error columns: <span id='errorElements'></span></p>");
+    out.println("<script>");
+    out.println("	  var errorElementNames = [];");
+    out.println("     const err_elements = document.querySelectorAll('.cellFeedback.error');");
+    out.println("     const errorCount = err_elements.length;");
+    out.println("     document.getElementById('errorCount').textContent=errorCount");
+    out.println("     if(errorCount>0){");
+    out.println("        var errorTitle = document.getElementById('errorHeader');  ");
+    out.println("        var errorMessage = document.createElement('p');  ");
+    out.println("        errorMessage.style.color='red';  ");
+    out.println("        errorMessage.innerText='Errors are preventing submission of this bulk import.';  ");
+    out.println("        errorTitle.insertAdjacentElement('afterend', errorMessage);   ");
+
+
+    out.println("     	for (var i = 0; i < err_elements.length; i ++ ){");
+    out.println("     		var errorElement = err_elements[i];");
+    out.println("     	  var table = errorElement.closest('table');");
+    out.println("     	  var headerRow = table.querySelector('tbody tr.headerRow');");
+    out.println("     	  var th = headerRow.children[errorElement.cellIndex].querySelector('th div span.tableFeedbackColumnHeader').innerHTML;");
+    out.println("     	  if(!errorElementNames.includes(th))errorElementNames.push(th); ");
+    out.println("     	}");
+    out.println("     	var errorList = document.getElementById('errorElements')");    
+    out.println("     	const ul = document.createElement('ul');");
+    out.println("     	errorElementNames.toString().split(',').forEach((item) => {");
+    out.println("          const li = document.createElement('li');");
+    out.println("     	   li.textContent = item;");
+    out.println("          ul.appendChild(li);");
+    out.println("     	});");
+    out.println("     	errorList.appendChild(ul);");
+    out.println("     }");
+    out.println("</script>");
+
+
 
     String uName = request.getUserPrincipal().getName();
     if (committing&&uName!=null) out.println("<p><a href=\"../encounters/searchResults.jsp?username="+uName+"\">Search encounters owned by current user \""+uName+"\"</a></p>");
@@ -894,10 +931,40 @@ public Encounter loadEncounter(Row row, ArrayList<Annotation> annotations, Strin
     if (sex!=null) enc.setSex(sex);
 
     String genus = getString(row, "Encounter.genus",colIndexMap, verbose, missingColumns, unusedColumns, feedback);
-    if (genus!=null) enc.setGenus(genus);
+    boolean hasGenus=false;
+    if (genus!=null && !genus.trim().equals("")) {
+    	hasGenus=true;
+    	enc.setGenus(genus.trim());
+    }
 
     String specificEpithet = getString(row, "Encounter.specificEpithet",colIndexMap, verbose, missingColumns, unusedColumns, feedback);
-    if (specificEpithet!=null) enc.setSpecificEpithet(specificEpithet);
+    boolean hasSpecificEpithet=false;
+    if (specificEpithet!=null && !specificEpithet.trim().equals("")) {
+    	hasSpecificEpithet=true;
+    	enc.setSpecificEpithet(specificEpithet.trim());
+    }
+
+    //start check for missing or unconfigured genus+species
+    if (!hasGenus) {
+        //mark genus empty
+    	feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap),"GENUS", row, "MISSING GENUS");
+    }
+    if (!hasSpecificEpithet) {
+        //mark specific epithet
+        feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap),"SPECIFIC EPITHET", row, "MISSING SPECIFIC EPITHET");
+    }
+    //now validate that a present genus and species value are supported
+    if(hasGenus && hasSpecificEpithet) {
+
+    	List<String> configuredSpecies = CommonConfiguration.getIndexedPropertyValues("genusSpecies", myShepherd.getContext());
+    	if(configuredSpecies!=null && configuredSpecies.size()>0 && configuredSpecies.toString().indexOf(enc.getTaxonomyString())<0) {
+    		//if bad values
+    		feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap), genus, row,"UNSUPPORTED VALUE: "+genus);
+    		feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap), specificEpithet, row, "UNSUPPORTED VALUE: "+specificEpithet);
+    	}
+
+    }
+    //end check for missing or unconfigured genus+species
 
     String submitterOrganization = getString(row, "Encounter.submitterOrganization",colIndexMap, verbose, missingColumns, unusedColumns, feedback);
     if (submitterOrganization!=null) enc.setSubmitterOrganization(submitterOrganization);

src/main/webapp/import/uploadFooter.jsp

@@ -52,8 +52,17 @@ if (!committing) {
 %>
 	<p>If you are adding many images and encounters (more than a couple hundred if each) this may take a while. You will be redirected to a status page as the process completes.</p>
 	<p><a onclick="sendAndRedirect('<%=uploadAction %>','<%=uuid %>')"><button id="commitButton" onclick="confirmCommit()">Commit these results.</button></a></p>
+	<script>
+		console.log("Commit count: "+errorCount);
+		if(errorCount>0){
+			var commitButton = document.getElementById('commitButton');
+			commitButton.setAttribute("disabled", "disabled");
+		}
+	</script>
 <%
 }
 %>
 
+
+
 </div></div><!-- container maincontent -->