Source code

001package org.hl7.fhir.r4.context;
002
003import java.io.File;
004import java.io.FileNotFoundException;
005import java.io.FileOutputStream;
006import java.io.FileWriter;
007import java.io.IOException;
008import java.io.OutputStreamWriter;
009import java.nio.charset.Charset;
010import java.text.DateFormat;
011import java.text.ParseException;
012import java.text.SimpleDateFormat;
013import java.util.ArrayList;
014import java.util.Date;
015import java.util.HashMap;
016import java.util.List;
017import java.util.Map;
018
019import org.apache.commons.codec.Charsets;
020import org.apache.commons.lang3.StringUtils;
021import org.hl7.fhir.exceptions.FHIRException;
022import org.hl7.fhir.exceptions.FHIRFormatError;
023import org.hl7.fhir.r4.context.IWorkerContext.ValidationResult;
024import org.hl7.fhir.r4.formats.IParser.OutputStyle;
025import org.hl7.fhir.r4.formats.JsonParser;
026import org.hl7.fhir.r4.model.CodeSystem.ConceptDefinitionComponent;
027import org.hl7.fhir.r4.model.CodeableConcept;
028import org.hl7.fhir.r4.model.Coding;
029import org.hl7.fhir.r4.model.UriType;
030import org.hl7.fhir.r4.model.ValueSet;
031import org.hl7.fhir.r4.model.ValueSet.ConceptSetComponent;
032import org.hl7.fhir.r4.model.ValueSet.ConceptSetFilterComponent;
033import org.hl7.fhir.r4.model.ValueSet.ValueSetExpansionContainsComponent;
034import org.hl7.fhir.r4.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
035import org.hl7.fhir.r4.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
036import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
037import org.hl7.fhir.utilities.TextFile;
038import org.hl7.fhir.utilities.Utilities;
039import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
040
041import com.google.gson.JsonElement;
042import com.google.gson.JsonNull;
043import com.google.gson.JsonObject;
044import com.google.gson.JsonPrimitive;
045
046/**
047 * This implements a two level cache. 
048 *  - a temporary cache for remmbering previous local operations
049 *  - a persistent cache for rembering tx server operations
050 *  
051 * the cache is a series of pairs: a map, and a list. the map is the loaded cache, the list is the persiistent cache, carefully maintained in order for version control consistency
052 * 
053 * @author graha
054 *
055 */
056public class TerminologyCache {
057  public static final boolean TRANSIENT = false;
058  public static final boolean PERMANENT = true;
059  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
060  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
061  private static final String BREAK = "####";
062
063  public class CacheToken {
064    private String name;
065    private String key;
066    private String request;
067    public void setName(String n) {
068      if (name == null)
069        name = n;
070      else if (!n.equals(name))
071        name = NAME_FOR_NO_SYSTEM;
072    }
073  }
074
075  private class CacheEntry {
076    private String request;
077    private boolean persistent;
078    private ValidationResult v;
079    private ValueSetExpansionOutcome e;
080  }
081  
082  private class NamedCache {
083    private String name; 
084    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
085    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
086  }
087  
088
089  private Object lock;
090  private String folder;
091  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
092  
093  // use lock from the context
094  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
095    super();
096    this.lock = lock;
097    this.folder = folder;
098    if (folder != null)
099      load();
100  }
101  
102  public CacheToken generateValidationToken(Coding code, ValueSet vs) {
103    CacheToken ct = new CacheToken();
104    if (code.hasSystem())
105      ct.name = getNameForSystem(code.getSystem());
106    else
107      ct.name = NAME_FOR_NO_SYSTEM;
108    JsonParser json = new JsonParser();
109    json.setOutputStyle(OutputStyle.PRETTY);
110    ValueSet vsc = getVSEssense(vs);
111    try {
112      ct.request = "{\"code\" : "+json.composeString(code, "code")+", \"valueSet\" :"+(vsc == null ? "null" : json.composeString(vsc))+"}";
113    } catch (IOException e) {
114      throw new Error(e);
115    }
116    ct.key = String.valueOf(hashNWS(ct.request));
117    return ct;
118  }
119
120  public CacheToken generateValidationToken(CodeableConcept code, ValueSet vs) {
121    CacheToken ct = new CacheToken();
122    for (Coding c : code.getCoding()) {
123      if (c.hasSystem())
124        ct.setName(getNameForSystem(c.getSystem()));
125    }
126    JsonParser json = new JsonParser();
127    json.setOutputStyle(OutputStyle.PRETTY);
128    ValueSet vsc = getVSEssense(vs);
129    try {
130      ct.request = "{\"code\" : "+json.composeString(code, "codeableConcept")+", \"valueSet\" :"+json.composeString(vsc)+"}";
131    } catch (IOException e) {
132      throw new Error(e);
133    }
134    ct.key = String.valueOf(hashNWS(ct.request));
135    return ct;
136  }
137  
138  public ValueSet getVSEssense(ValueSet vs) {
139    if (vs == null)
140      return null;
141    ValueSet vsc = new ValueSet();
142    vsc.setCompose(vs.getCompose());
143    if (vs.hasExpansion()) {
144      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
145      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
146    }
147    return vsc;
148  }
149
150  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
151    CacheToken ct = new CacheToken();
152    ValueSet vsc = getVSEssense(vs);
153    for (ConceptSetComponent inc : vs.getCompose().getInclude())
154      if (inc.hasSystem())
155        ct.setName(getNameForSystem(inc.getSystem()));
156    for (ConceptSetComponent inc : vs.getCompose().getExclude())
157      if (inc.hasSystem())
158        ct.setName(getNameForSystem(inc.getSystem()));
159    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
160      if (inc.hasSystem())
161        ct.setName(getNameForSystem(inc.getSystem()));
162    JsonParser json = new JsonParser();
163    json.setOutputStyle(OutputStyle.PRETTY);
164    try {
165      ct.request = "{\"hierarchical\" : "+(heirarchical ? "true" : "false")+", \"valueSet\" :"+json.composeString(vsc)+"}\r\n";
166    } catch (IOException e) {
167      throw new Error(e);
168    }
169    ct.key = String.valueOf(hashNWS(ct.request));
170    return ct;
171  }
172
173  private String getNameForSystem(String system) {
174    if (system.equals("http://snomed.info/sct"))
175      return "snomed";
176    if (system.equals("http://www.nlm.nih.gov/research/umls/rxnorm"))
177      return "rxnorm";
178    if (system.equals("http://loinc.org"))
179      return "loinc";
180    if (system.equals("http://unitsofmeasure.org"))
181      return "ucum";
182    if (system.startsWith("http://hl7.org/fhir/sid/"))
183      return system.substring(24).replace("/", "");
184    if (system.startsWith("urn:iso:std:iso:"))
185      return "iso"+system.substring(16).replace(":", "");
186    if (system.startsWith("http://terminology.hl7.org/CodeSystem/"))
187      return system.substring(38).replace("/", "");
188    if (system.startsWith("http://hl7.org/fhir/"))
189      return system.substring(20).replace("/", "");
190    if (system.equals("urn:ietf:bcp:47"))
191      return "lang";
192    if (system.equals("urn:ietf:bcp:13"))
193      return "mimetypes";
194    if (system.equals("urn:iso:std:iso:11073:10101"))
195      return "11073";
196    if (system.equals("http://dicom.nema.org/resources/ontology/DCM"))
197      return "dicom";
198    return system.replace("/", "_").replace(":", "_");
199  }
200
201  public NamedCache getNamedCache(CacheToken cacheToken) {
202    NamedCache nc = caches.get(cacheToken.name);
203    if (nc == null) {
204      nc = new NamedCache();
205      nc.name = cacheToken.name;
206      caches.put(nc.name, nc);
207    }
208    return nc;
209  }
210  
211  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
212    synchronized (lock) {
213      NamedCache nc = getNamedCache(cacheToken);
214      CacheEntry e = nc.map.get(cacheToken.key);
215      if (e == null)
216        return null;
217      else
218        return e.e;
219    }
220  }
221
222  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
223    synchronized (lock) {      
224      NamedCache nc = getNamedCache(cacheToken);
225      CacheEntry e = new CacheEntry();
226      e.request = cacheToken.request;
227      e.persistent = persistent;
228      e.e = res;
229      store(cacheToken, persistent, nc, e);
230    }    
231  }
232
233  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
234    boolean n = nc.map.containsKey(cacheToken.key);
235    nc.map.put(cacheToken.key, e);
236    if (persistent) {
237      if (n) {
238        for (int i = nc.list.size()- 1; i>= 0; i--) {
239          if (nc.list.get(i).request.equals(e.request)) {
240            nc.list.remove(i);
241          }
242        }
243      }
244      nc.list.add(e);
245      save(nc);  
246    }
247  }
248
249  public ValidationResult getValidation(CacheToken cacheToken) {
250    synchronized (lock) {
251      NamedCache nc = getNamedCache(cacheToken);
252      CacheEntry e = nc.map.get(cacheToken.key);
253      if (e == null)
254        return null;
255      else
256        return e.v;
257    }
258  }
259
260  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
261    synchronized (lock) {      
262      NamedCache nc = getNamedCache(cacheToken);
263      CacheEntry e = new CacheEntry();
264      e.request = cacheToken.request;
265      e.persistent = persistent;
266      e.v = res;
267      store(cacheToken, persistent, nc, e);
268    }    
269  }
270
271  
272  // persistence
273  
274  public void save() {
275    
276  }
277  
278  private void save(NamedCache nc) {
279    if (folder == null)
280      return;
281    
282    try {
283      OutputStreamWriter sw = new OutputStreamWriter(new FileOutputStream(Utilities.path(folder, nc.name+".cache")), "UTF-8");
284      sw.write(ENTRY_MARKER+"\r\n");
285      JsonParser json = new JsonParser();
286      json.setOutputStyle(OutputStyle.PRETTY);
287      for (CacheEntry ce : nc.list) {
288        sw.write(ce.request.trim());
289        sw.write(BREAK+"\r\n");
290        if (ce.e != null) {
291          sw.write("e: {\r\n");
292          if (ce.e.getValueset() != null)
293            sw.write("  \"valueSet\" : "+json.composeString(ce.e.getValueset()).trim()+",\r\n");
294          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.e.getError()).trim()+"\"\r\n}\r\n");
295        } else {
296          sw.write("v: {\r\n");
297          sw.write("  \"display\" : \""+Utilities.escapeJson(ce.v.getDisplay()).trim()+"\",\r\n");
298          sw.write("  \"severity\" : "+(ce.v.getSeverity() == null ? "null" : "\""+ce.v.getSeverity().toCode().trim()+"\"")+",\r\n");
299          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.v.getMessage()).trim()+"\"\r\n}\r\n");
300        }
301        sw.write(ENTRY_MARKER+"\r\n");
302      }      
303      sw.close();
304    } catch (Exception e) {
305      System.out.println("error saving "+nc.name+": "+e.getMessage());
306    }
307  }
308
309  private void load() throws FHIRException {
310    for (String fn : new File(folder).list()) {
311      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
312        try {
313          //  System.out.println("Load "+fn);
314          String title = fn.substring(0, fn.lastIndexOf("."));
315          NamedCache nc = new NamedCache();
316          nc.name = title;
317          caches.put(title, nc);
318          String src = TextFile.fileToString(Utilities.path(folder, fn));
319          if (src.startsWith("?"))
320            src = src.substring(1);
321          int i = src.indexOf(ENTRY_MARKER); 
322          while (i > -1) {
323            String s = src.substring(0, i);
324            src = src.substring(i+ENTRY_MARKER.length()+1);
325            i = src.indexOf(ENTRY_MARKER);
326            if (!Utilities.noString(s)) {
327              int j = s.indexOf(BREAK);
328              String q = s.substring(0, j);
329              String p = s.substring(j+BREAK.length()+1).trim();
330              CacheEntry ce = new CacheEntry();
331              ce.persistent = true;
332              ce.request = q;
333              boolean e = p.charAt(0) == 'e';
334              p = p.substring(3);
335              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
336              String error = loadJS(o.get("error"));
337              if (e) {
338                if (o.has("valueSet"))
339                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
340                else
341                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
342              } else {
343                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null :  IssueSeverity.fromCode(o.get("severity").getAsString());
344                String display = loadJS(o.get("display"));
345                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
346              }
347              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
348              nc.list.add(ce);
349            }
350          }        
351        } catch (Exception e) {
352          throw new FHIRException("Error loading "+fn+": "+e.getMessage(), e);
353        }
354      }
355    }
356  }
357  
358  private String loadJS(JsonElement e) {
359    if (e == null)
360      return null;
361    if (!(e instanceof JsonPrimitive))
362      return null;
363    String s = e.getAsString();
364    if ("".equals(s))
365      return null;
366    return s;
367  }
368
369  private String hashNWS(String s) {
370    return String.valueOf(s.replace("\r", "").replace("\n", "").replace(" ", "").hashCode());
371  }
372
373  // management
374  
375  public TerminologyCache copy() {
376    // TODO Auto-generated method stub
377    return null;
378  }
379  
380  public String summary(ValueSet vs) {
381    if (vs == null)
382      return "null";
383    
384    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
385    for (ConceptSetComponent cc : vs.getCompose().getInclude())
386      b.append("Include "+getIncSummary(cc));
387    for (ConceptSetComponent cc : vs.getCompose().getExclude())
388      b.append("Exclude "+getIncSummary(cc));
389    return b.toString();
390  }
391
392  private String getIncSummary(ConceptSetComponent cc) {
393    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
394    for (UriType vs : cc.getValueSet())
395      b.append(vs.asStringValue());
396    String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
397    String system = cc.getSystem();
398    if (cc.hasConcept())
399      return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
400    if (cc.hasFilter()) {
401      String s = "";
402      for (ConceptSetFilterComponent f : cc.getFilter()) {
403        if (!Utilities.noString(s))
404          s = s + " & ";
405        s = s + f.getProperty()+" "+f.getOp().toCode()+" "+f.getValue();
406      }
407      return "from "+system+" where "+s+vsd;
408    }
409    return "All codes from "+system+vsd;
410  }
411
412  public String summary(Coding code) {
413    return code.getSystem()+"#"+code.getCode()+": \""+code.getDisplay()+"\"";
414  }
415
416
417  public String summary(CodeableConcept code) {
418    StringBuilder b = new StringBuilder();
419    b.append("{");
420    boolean first = true;
421    for (Coding c : code.getCoding()) {
422      if (first) first = false; else b.append(",");
423      b.append(summary(c));
424    }
425    b.append("}: \"");
426    b.append(code.getText());
427    b.append("\"");
428    return b.toString();
429  }
430  
431}