From f32540fea3ef1e7acfc35fd80ac2aaa295ab7ec3 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sat, 30 Mar 2024 22:05:00 -0700 Subject: [PATCH 1/5] Report error on empty species and unconfigured species --- .../org/ecocean/importutils/RowFeedback.java | 14 ++++---- .../servlet/importer/StandardImport.java | 34 +++++++++++++++++-- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/ecocean/importutils/RowFeedback.java b/src/main/java/org/ecocean/importutils/RowFeedback.java index 76cb8880d2..2fac506fa3 100644 --- a/src/main/java/org/ecocean/importutils/RowFeedback.java +++ b/src/main/java/org/ecocean/importutils/RowFeedback.java @@ -66,12 +66,14 @@ public void logParseError(int colNum, Object value, Row row) { CellFeedback cellFeedback = this.cells[colNum]; System.out.println("Setting ERROR value on OLD CellFeedback for col "+colNum+" val "+String.valueOf(value)+" row "+row.getRowNum()); // I think we can assume a BLANK or NULL cell doesn't need to get overwritten with an error. - if (!cellFeedback.isBlank()) { - this.cells[colNum].setSuccess(false); - //TODO replace this universal NOT FOUND for an overwrite with something specific. - this.cells[colNum].setValueString(value+" NOT FOUND"); - } - } else { + //if (!cellFeedback.isBlank()) { + this.cells[colNum].setSuccess(false); + this.cells[colNum].setIsBlank(false); + //TODO replace this universal NOT FOUND for an overwrite with something specific. + this.cells[colNum].setValueString(value+" NOT FOUND"); + //} + } + else { System.out.println("Setting ERROR value on NEW CellFeedback for col "+colNum+" val "+String.valueOf(value)+" row "+row.getRowNum()); this.cells[colNum] = new CellFeedback(value, false, false); } diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index f50171bf9c..b5ba7715dc 100755 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -894,10 +894,40 @@ public Encounter loadEncounter(Row row, ArrayList annotations, Strin if (sex!=null) enc.setSex(sex); String genus = getString(row, "Encounter.genus",colIndexMap, verbose, missingColumns, unusedColumns, feedback); - if (genus!=null) enc.setGenus(genus); + boolean hasGenus=false; + if (genus!=null && !genus.trim().equals("")) { + hasGenus=true; + enc.setGenus(genus.trim()); + } String specificEpithet = getString(row, "Encounter.specificEpithet",colIndexMap, verbose, missingColumns, unusedColumns, feedback); - if (specificEpithet!=null) enc.setSpecificEpithet(specificEpithet); + boolean hasSpecificEpithet=false; + if (specificEpithet!=null && !specificEpithet.trim().equals("")) { + hasSpecificEpithet=true; + enc.setSpecificEpithet(specificEpithet.trim()); + } + + //start check for missing or unconfigured genus+species + if (!hasGenus) { + //mark genus empty + feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap),"GENUS", row); + } + if (!hasSpecificEpithet) { + //mark specific epithet + feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap),"SPECIFIC EPITHET", row); + } + //now validate that a present genus and species value are supported + if(hasGenus && hasSpecificEpithet) { + + List configuredSpecies = CommonConfiguration.getIndexedPropertyValues("genusSpecies", myShepherd.getContext()); + if(configuredSpecies!=null && configuredSpecies.size()>0 && configuredSpecies.toString().indexOf(enc.getTaxonomyString())<0) { + //if bad values + feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap), genus, row); + feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap), specificEpithet, row); + } + + } + //end check for missing or unconfigured genus+species String submitterOrganization = getString(row, "Encounter.submitterOrganization",colIndexMap, verbose, missingColumns, unusedColumns, feedback); if (submitterOrganization!=null) enc.setSubmitterOrganization(submitterOrganization); From 60463a3ddd9e018163afee91ae7df81e47a2dd0b Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sun, 31 Mar 2024 11:08:03 -0700 Subject: [PATCH 2/5] Add custom message passthrough for better descriptions of what is wrong with a field --- .../java/org/ecocean/importutils/RowFeedback.java | 13 +++++++++++-- .../org/ecocean/importutils/TabularFeedback.java | 10 +++++++--- .../ecocean/servlet/importer/StandardImport.java | 8 ++++---- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/ecocean/importutils/RowFeedback.java b/src/main/java/org/ecocean/importutils/RowFeedback.java index 2fac506fa3..cb8fb5381e 100644 --- a/src/main/java/org/ecocean/importutils/RowFeedback.java +++ b/src/main/java/org/ecocean/importutils/RowFeedback.java @@ -58,8 +58,12 @@ public void logParseValue(int colNum, Object value, Row row) { } } } - + public void logParseError(int colNum, Object value, Row row) { + logParseError(colNum, value, row, null); + } + + public void logParseError(int colNum, Object value, Row row, String exactMessageToDisplay) { try { if (!committing) { if (colNum annotations, Strin //start check for missing or unconfigured genus+species if (!hasGenus) { //mark genus empty - feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap),"GENUS", row); + feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap),"GENUS", row, "MISSING GENUS"); } if (!hasSpecificEpithet) { //mark specific epithet - feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap),"SPECIFIC EPITHET", row); + feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap),"SPECIFIC EPITHET", row, "MISSING SPECIFIC EPITHET"); } //now validate that a present genus and species value are supported if(hasGenus && hasSpecificEpithet) { @@ -922,8 +922,8 @@ public Encounter loadEncounter(Row row, ArrayList annotations, Strin List configuredSpecies = CommonConfiguration.getIndexedPropertyValues("genusSpecies", myShepherd.getContext()); if(configuredSpecies!=null && configuredSpecies.size()>0 && configuredSpecies.toString().indexOf(enc.getTaxonomyString())<0) { //if bad values - feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap), genus, row); - feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap), specificEpithet, row); + feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap), genus, row,"UNCONFIGURED VALUE: "+genus); + feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap), specificEpithet, row, "UNCONFIGURED VALUE: "+specificEpithet); } } From 9b781f2ab47ae774827c5cc5726697feb67a2d83 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Sun, 31 Mar 2024 18:30:46 -0700 Subject: [PATCH 3/5] Provide a count of errors in the import --- .../ecocean/servlet/importer/StandardImport.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index fd75353834..bba712eaf9 100755 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -551,7 +551,7 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, } out.println("
"); // half page bootstrap column - out.println("

Import Overview:

"); + out.println("

Import Overview

"); out.println("
    "); out.println("
  • Excel File Name: "+filename+"
    • "); out.println("
  • Excel File Successfully Found = "+dataFound+"
    • "); @@ -560,8 +560,17 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, out.println("
  • Excel Columns = "+cols+"
    • "); //out.println("
  • Last col num = "+lastColNum+"
    • "); out.println("
  • Trial Run: "+!committing+"
    • "); - out.println("
"); + + out.println("

Errors

"); + out.println("

Number of errors:

"); + out.println(""); + + String uName = request.getUserPrincipal().getName(); if (committing&&uName!=null) out.println("

Search encounters owned by current user \""+uName+"\"

"); From 275393221d4ad90ebbebb2fc7eb55bc6a77408ce Mon Sep 17 00:00:00 2001 From: holmbergius Date: Tue, 2 Apr 2024 18:10:53 -0700 Subject: [PATCH 4/5] List error columns --- .../servlet/importer/StandardImport.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index bba712eaf9..51cb392ac6 100755 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -44,6 +44,8 @@ import org.json.JSONArray; import org.json.JSONObject; + + import javax.servlet.RequestDispatcher; import javax.servlet.ServletConfig; import javax.servlet.ServletContext; @@ -564,10 +566,30 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, out.println("

Errors

"); out.println("

Number of errors:

"); + out.println("

Error columns:

"); out.println(""); @@ -931,8 +953,8 @@ public Encounter loadEncounter(Row row, ArrayList annotations, Strin List configuredSpecies = CommonConfiguration.getIndexedPropertyValues("genusSpecies", myShepherd.getContext()); if(configuredSpecies!=null && configuredSpecies.size()>0 && configuredSpecies.toString().indexOf(enc.getTaxonomyString())<0) { //if bad values - feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap), genus, row,"UNCONFIGURED VALUE: "+genus); - feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap), specificEpithet, row, "UNCONFIGURED VALUE: "+specificEpithet); + feedback.logParseError(getColIndexFromColName("Encounter.genus", colIndexMap), genus, row,"UNSUPPORTED VALUE: "+genus); + feedback.logParseError(getColIndexFromColName("Encounter.specificEpithet", colIndexMap), specificEpithet, row, "UNSUPPORTED VALUE: "+specificEpithet); } } From 5ace7a20874f408a72853e2aff07ef6d49587e66 Mon Sep 17 00:00:00 2001 From: holmbergius Date: Tue, 2 Apr 2024 22:08:37 -0700 Subject: [PATCH 5/5] Block submission and list error columns --- .../servlet/importer/StandardImport.java | 36 +++++++++++-------- src/main/webapp/import/uploadFooter.jsp | 9 +++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/ecocean/servlet/importer/StandardImport.java b/src/main/java/org/ecocean/servlet/importer/StandardImport.java index 51cb392ac6..ca5b80b738 100755 --- a/src/main/java/org/ecocean/servlet/importer/StandardImport.java +++ b/src/main/java/org/ecocean/servlet/importer/StandardImport.java @@ -564,32 +564,38 @@ public void doImport(String filename, File dataFile, HttpServletRequest request, out.println("
  • Trial Run: "+!committing+"
    • "); out.println(""); - out.println("

    Errors

    "); + out.println("

    Errors

    "); out.println("

    Number of errors:

    "); out.println("

    Error columns:

    "); out.println(""); diff --git a/src/main/webapp/import/uploadFooter.jsp b/src/main/webapp/import/uploadFooter.jsp index 8946b6c163..0f9b95fc6f 100644 --- a/src/main/webapp/import/uploadFooter.jsp +++ b/src/main/webapp/import/uploadFooter.jsp @@ -52,8 +52,17 @@ if (!committing) { %>

    If you are adding many images and encounters (more than a couple hundred if each) this may take a while. You will be redirected to a status page as the process completes.

    + <% } %> + +