Merge pull request #1494 from hapifhir/2023-11-gg-misc1

2023 11 gg misc1
hapifhir · Nov 24, 2023 · 7f42c7a · 7f42c7a
2 parents 8fe88a3 + 82f0fc9
commit 7f42c7a
Show file tree

Hide file tree

Showing 81 changed files with 6,027 additions and 521 deletions.
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
@@ -1,7 +1,18 @@
 ## Validator Changes
 
-* no changes
+* Rework bundle references validation for R4+ - this is a *significant* change - many existing bundles that were previously erroneously passing will now fail
+* #1488 - don't fail on erroneously repeating elements
+* Fix problem creating CDA type discriminators
+* Fix problem with R3 expansion
+* Add support for CCDA .hasTemplateIdOf(canonical)
+* Add support for NZ IPS
+
 
 ## Other code changes
 
-* no changes
+* Start work on making IWorkerContext a versioned API
+  * Add fhirVersion to R5 Base and IWorkerContext methods
+  * move IContextResourceLoader, ValidationResult and CodingValidationRequest out of IWorkerContext to their own clasess
+* Fix up VSAC import for large value sets
+* fix FHIRPath cda tests for empty package cache
+* Fix issue where markdown with multiple characters was being cut off sometimes when rendering profiles
diff --git a/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java b/org.hl7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/misc/VSACImporter.java
@@ -2,11 +2,15 @@
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.net.URISyntaxException;
 import java.text.ParseException;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.hl7.fhir.exceptions.FHIRException;
@@ -24,9 +28,12 @@
 import org.hl7.fhir.r4.terminologies.JurisdictionUtilities;
 import org.hl7.fhir.utilities.CSVReader;
 import org.hl7.fhir.utilities.Utilities;
+import org.hl7.fhir.utilities.json.model.JsonArray;
+import org.hl7.fhir.utilities.json.model.JsonObject;
+import org.hl7.fhir.utilities.json.model.JsonProperty;
 
 public class VSACImporter extends OIDBasedValueSetImporter {
-
+  
   public VSACImporter() throws FHIRException, IOException {
     super();
     init();
@@ -45,54 +52,25 @@ private void process(String source, String dest, String apiKey, boolean onlyNew)
     FHIRToolingClient fhirToolingClient = new FHIRToolingClient("https://cts.nlm.nih.gov/fhir", "fhir/vsac");
     fhirToolingClient.setUsername("apikey");
     fhirToolingClient.setPassword(apiKey);
-    fhirToolingClient.setTimeout(120000);
+    fhirToolingClient.setTimeoutNormal(6000);
 
     CapabilityStatement cs = fhirToolingClient.getCapabilitiesStatement();
     JsonParser json = new JsonParser();
     json.setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path("[tmp]", "vsac-capability-statmenet.json")), cs);
-    
-    int i = 0;
-    int j = 0;
+
+    System.out.println("Loading");
+    List<String> oids = new ArrayList<>();
     while (csv.line()) {
       String oid = csv.cell("OID");
+      oids.add(oid);
+    }
+    Collections.sort(oids);
+    System.out.println("Go: "+oids.size()+" oids");
+    int i = 0;
+    int j = 0;
+    for (String oid : oids) {
       try {
-        if (!onlyNew || !(new File(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) {
-          ValueSet vs = fhirToolingClient.read(ValueSet.class, oid);
-          try {
-            ValueSet vse = fhirToolingClient.expandValueset(vs.getUrl(), null);
-            vs.setExpansion(vse.getExpansion());
-            j++;
-          } catch (Exception e) {
-            errs.put(oid, "Expansion: " +e.getMessage());
-            System.out.println(e.getMessage());
-          }
-          while (isIncomplete(vs.getExpansion())) {
-            Parameters p = new Parameters();
-            p.addParameter("offset", vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue());
-            ValueSet vse = fhirToolingClient.expandValueset(vs.getUrl(), p);    
-            vs.getExpansion().getContains().addAll(vse.getExpansion().getContains());
-            vs.getExpansion().setParameter(vse.getExpansion().getParameter());
-          }
-          vs.getExpansion().setOffsetElement(null);
-          vs.getExpansion().getParameter().clear();
-
-
-          if (vs.hasTitle()) {
-            if (vs.getTitle().equals(vs.getDescription())) {
-              vs.setTitle(vs.getName());              
-            } else {
-//              System.out.println(oid);
-//              System.out.println("  name: "+vs.getName());
-//              System.out.println("  title: "+vs.getTitle());
-//              System.out.println("  desc: "+vs.getDescription());
-            }
-          } else {
-            vs.setTitle(vs.getName());
-          }
-          vs.setName(makeValidName(vs.getName()));
-          JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US");
-          new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
-        }
+        j = processOid(dest, onlyNew, errs, fhirToolingClient, j, oid.trim());
         i++;
         if (i % 100 == 0) {
           System.out.println(":"+i+" ("+j+")");
@@ -110,6 +88,62 @@ private void process(String source, String dest, String apiKey, boolean onlyNew)
     System.out.println("Done. " + i + " ValueSets");
   }
 
+  private int processOid(String dest, boolean onlyNew, Map<String, String> errs, FHIRToolingClient fhirToolingClient, int j, String oid)
+      throws IOException, InterruptedException, FileNotFoundException {
+    if (!onlyNew || !(new File(Utilities.path(dest, "ValueSet-" + oid + ".json")).exists())) {
+      ValueSet vs = null;
+      try {
+        vs = fhirToolingClient.read(ValueSet.class, oid);
+      } catch (Exception e) {
+        errs.put(oid, "Read: " +e.getMessage());
+        System.out.println("Read "+oid+" failed: "+e.getMessage());
+      }
+      if (vs != null) {
+        try {
+          ValueSet vse = fhirToolingClient.expandValueset(vs.getUrl(), null);
+          vs.setExpansion(vse.getExpansion());
+          j++;
+        } catch (Exception e) {
+          errs.put(oid, "Expansion: " +e.getMessage());
+          System.out.println("Expand "+oid+" failed: "+e.getMessage());
+        }
+        while (isIncomplete(vs.getExpansion())) {
+          Parameters p = new Parameters();
+          int offset = vs.getExpansion().getParameter("offset").getValueIntegerType().getValue() + vs.getExpansion().getParameter("count").getValueIntegerType().getValue();
+          p.addParameter("offset", offset);
+          try {
+            ValueSet vse = fhirToolingClient.expandValueset(vs.getUrl(), p);    
+            vs.getExpansion().getContains().addAll(vse.getExpansion().getContains());
+            vs.getExpansion().setParameter(vse.getExpansion().getParameter());
+          } catch (Exception e2) {
+            errs.put(oid, "Expansion: " +e2.getMessage()+" @ "+offset);
+            System.out.println("Expand "+oid+" @ "+offset+" failed: "+e2.getMessage());
+          } 
+        }
+        vs.getExpansion().setOffsetElement(null);
+        vs.getExpansion().getParameter().clear();
+
+
+        if (vs.hasTitle()) {
+          if (vs.getTitle().equals(vs.getDescription())) {
+            vs.setTitle(vs.getName());              
+          } else {
+            //              System.out.println(oid);
+            //              System.out.println("  name: "+vs.getName());
+            //              System.out.println("  title: "+vs.getTitle());
+            //              System.out.println("  desc: "+vs.getDescription());
+          }
+        } else {
+          vs.setTitle(vs.getName());
+        }
+        vs.setName(makeValidName(vs.getName()));
+        JurisdictionUtilities.setJurisdictionCountry(vs.getJurisdiction(), "US");
+        new JsonParser().setOutputStyle(OutputStyle.PRETTY).compose(new FileOutputStream(Utilities.path(dest, "ValueSet-" + oid + ".json")), vs);
+      }
+    }
+    return j;
+  }
+
   private boolean isIncomplete(ValueSetExpansionComponent expansion) {
     IntegerType c = expansion.getParameter("count").getValueIntegerType();
     IntegerType offset = expansion.getParameter("offset").getValueIntegerType();

diff --git a/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR2.java b/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR2.java
@@ -125,8 +125,8 @@ public Parameters validateVS(Parameters pin) throws FHIRException {
   }
 
   @Override
-  public ITerminologyClient setTimeout(int i) {
-    client.setTimeout(i);
+  public ITerminologyClient setTimeoutFactor(int i) {
+    client.setTimeoutFactor(i);
     return this;
   }
 

diff --git a/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR3.java b/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR3.java
@@ -123,8 +123,8 @@ public Parameters validateVS(Parameters pin) throws FHIRException {
   }
 
   @Override
-  public ITerminologyClient setTimeout(int i) {
-    client.setTimeout(i);
+  public ITerminologyClient setTimeoutFactor(int i) {
+    client.setTimeoutFactor(i);
     return this;
   }
 

diff --git a/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR4.java b/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR4.java
@@ -133,8 +133,8 @@ public Parameters validateVS(Parameters pin) throws FHIRException {
   }
 
   @Override
-  public ITerminologyClient setTimeout(int i) {
-    client.setTimeout(i);
+  public ITerminologyClient setTimeoutFactor(int i) {
+    client.setTimeoutFactor(i);
     return this;
   }
 

diff --git a/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR5.java b/...7.fhir.convertors/src/main/java/org/hl7/fhir/convertors/txClient/TerminologyClientR5.java
@@ -117,8 +117,8 @@ public Parameters validateVS(Parameters pin) {
   }
 
   @Override
-  public ITerminologyClient setTimeout(int i) {
-    client.setTimeout(i);
+  public ITerminologyClient setTimeoutFactor(int i) {
+    client.setTimeoutFactor(i);
     return this;
   }
 

diff --git a/org.hl7.fhir.convertors/src/test/resources/extension_r3_dt_base.json b/org.hl7.fhir.convertors/src/test/resources/extension_r3_dt_base.json
@@ -21,7 +21,7 @@
       "telecom": [
         {
           "system": "url",
-          "value": "http://hl7.org/special/committees/FHIR"
+          "value": "http://www.hl7.org/Special/committees/FHIR"
         }
       ]
     }

diff --git a/org.hl7.fhir.convertors/src/test/resources/extension_r3_dt_path.json b/org.hl7.fhir.convertors/src/test/resources/extension_r3_dt_path.json
@@ -21,7 +21,7 @@
       "telecom": [
         {
           "system": "url",
-          "value": "http://hl7.org/special/committees/FHIR"
+          "value": "http://www.hl7.org/Special/committees/FHIR"
         }
       ]
     }

diff --git a/org.hl7.fhir.convertors/src/test/resources/extension_r3_res_base.json b/org.hl7.fhir.convertors/src/test/resources/extension_r3_res_base.json
@@ -22,7 +22,7 @@
       "telecom": [
         {
           "system": "url",
-          "value": "http://hl7.org/special/committees/fhir.htm"
+          "value": "http://www.hl7.org/Special/committees/fhir.htm"
         }
       ]
     }

diff --git a/org.hl7.fhir.convertors/src/test/resources/extension_r3_res_path.json b/org.hl7.fhir.convertors/src/test/resources/extension_r3_res_path.json
@@ -22,7 +22,7 @@
       "telecom": [
         {
           "system": "url",
-          "value": "http://hl7.org/special/committees/fhir.htm"
+          "value": "http://www.hl7.org/Special/committees/fhir.htm"
         }
       ]
     }