001package org.hl7.fhir.r5.context;
002
003/*-
004 * #%L
005 * org.hl7.fhir.r5
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.File;
025import java.io.FileNotFoundException;
026import java.io.FileOutputStream;
027import java.io.IOException;
028import java.io.OutputStreamWriter;
029import java.util.ArrayList;
030import java.util.EnumSet;
031import java.util.HashMap;
032import java.util.List;
033import java.util.Map;
034
035import org.hl7.fhir.exceptions.FHIRException;
036import org.hl7.fhir.r5.context.IWorkerContext.ValidationResult;
037import org.hl7.fhir.r5.formats.IParser.OutputStyle;
038import org.hl7.fhir.r5.formats.JsonParser;
039import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
040import org.hl7.fhir.r5.model.CodeableConcept;
041import org.hl7.fhir.r5.model.Coding;
042import org.hl7.fhir.r5.model.UriType;
043import org.hl7.fhir.r5.model.ValueSet;
044import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
045import org.hl7.fhir.r5.model.ValueSet.ConceptSetFilterComponent;
046import org.hl7.fhir.r5.model.ValueSet.ValueSetExpansionContainsComponent;
047import org.hl7.fhir.r5.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
048import org.hl7.fhir.r5.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
049import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
050import org.hl7.fhir.utilities.TextFile;
051import org.hl7.fhir.utilities.Utilities;
052import org.hl7.fhir.utilities.validation.ValidationOptions;
053import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
054
055import com.google.gson.JsonElement;
056import com.google.gson.JsonNull;
057import com.google.gson.JsonObject;
058import com.google.gson.JsonPrimitive;
059
060/**
061 * This implements a two level cache. 
062 *  - a temporary cache for remmbering previous local operations
063 *  - a persistent cache for rembering tx server operations
064 *  
065 * the cache is a series of pairs: a map, and a list. the map is the loaded cache, the list is the persiistent cache, carefully maintained in order for version control consistency
066 * 
067 * @author graha
068 *
069 */
070public class TerminologyCache {
071  public static final boolean TRANSIENT = false;
072  public static final boolean PERMANENT = true;
073  private static final String NAME_FOR_NO_SYSTEM = "all-systems";
074  private static final String ENTRY_MARKER = "-------------------------------------------------------------------------------------";
075  private static final String BREAK = "####";
076
077  public class CacheToken {
078    private String name;
079    private String key;
080    private String request;
081    public void setName(String n) {
082      if (name == null)
083        name = n;
084      else if (!n.equals(name))
085        name = NAME_FOR_NO_SYSTEM;
086    }
087  }
088
089  private class CacheEntry {
090    private String request;
091    private boolean persistent;
092    private ValidationResult v;
093    private ValueSetExpansionOutcome e;
094  }
095  
096  private class NamedCache {
097    private String name; 
098    private List<CacheEntry> list = new ArrayList<CacheEntry>(); // persistent entries
099    private Map<String, CacheEntry> map = new HashMap<String, CacheEntry>();
100  }
101  
102
103  private Object lock;
104  private String folder;
105  private Map<String, NamedCache> caches = new HashMap<String, NamedCache>();
106  
107  // use lock from the context
108  public TerminologyCache(Object lock, String folder) throws FileNotFoundException, IOException, FHIRException {
109    super();
110    this.lock = lock;
111    this.folder = folder;
112    if (folder != null)
113      load();
114  }
115  
116  public CacheToken generateValidationToken(ValidationOptions options, Coding code, ValueSet vs) {
117    CacheToken ct = new CacheToken();
118    if (code.hasSystem())
119      ct.name = getNameForSystem(code.getSystem());
120    else
121      ct.name = NAME_FOR_NO_SYSTEM;
122    JsonParser json = new JsonParser();
123    json.setOutputStyle(OutputStyle.PRETTY);
124    ValueSet vsc = getVSEssense(vs);
125    try {
126      ct.request = "{\"code\" : "+json.composeString(code, "code")+", \"valueSet\" :"+(vsc == null ? "null" : json.composeString(vsc))+(options == null ? "" : ", "+options.toJson())+"}";
127    } catch (IOException e) {
128      throw new Error(e);
129    }
130    ct.key = String.valueOf(hashNWS(ct.request));
131    return ct;
132  }
133
134  public CacheToken generateValidationToken(ValidationOptions options, CodeableConcept code, ValueSet vs) {
135    CacheToken ct = new CacheToken();
136    for (Coding c : code.getCoding()) {
137      if (c.hasSystem())
138        ct.setName(getNameForSystem(c.getSystem()));
139    }
140    JsonParser json = new JsonParser();
141    json.setOutputStyle(OutputStyle.PRETTY);
142    ValueSet vsc = getVSEssense(vs);
143    try {
144      ct.request = "{\"code\" : "+json.composeString(code, "codeableConcept")+", \"valueSet\" :"+json.composeString(vsc)+(options == null ? "" : ", "+options.toJson())+"}";
145    } catch (IOException e) {
146      throw new Error(e);
147    }
148    ct.key = String.valueOf(hashNWS(ct.request));
149    return ct;
150  }
151  
152  public ValueSet getVSEssense(ValueSet vs) {
153    if (vs == null)
154      return null;
155    ValueSet vsc = new ValueSet();
156    vsc.setCompose(vs.getCompose());
157    if (vs.hasExpansion()) {
158      vsc.getExpansion().getParameter().addAll(vs.getExpansion().getParameter());
159      vsc.getExpansion().getContains().addAll(vs.getExpansion().getContains());
160    }
161    return vsc;
162  }
163
164  public CacheToken generateExpandToken(ValueSet vs, boolean heirarchical) {
165    CacheToken ct = new CacheToken();
166    ValueSet vsc = getVSEssense(vs);
167    for (ConceptSetComponent inc : vs.getCompose().getInclude())
168      if (inc.hasSystem())
169        ct.setName(getNameForSystem(inc.getSystem()));
170    for (ConceptSetComponent inc : vs.getCompose().getExclude())
171      if (inc.hasSystem())
172        ct.setName(getNameForSystem(inc.getSystem()));
173    for (ValueSetExpansionContainsComponent inc : vs.getExpansion().getContains())
174      if (inc.hasSystem())
175        ct.setName(getNameForSystem(inc.getSystem()));
176    JsonParser json = new JsonParser();
177    json.setOutputStyle(OutputStyle.PRETTY);
178    try {
179      ct.request = "{\"hierarchical\" : "+(heirarchical ? "true" : "false")+", \"valueSet\" :"+json.composeString(vsc)+"}\r\n";
180    } catch (IOException e) {
181      throw new Error(e);
182    }
183    ct.key = String.valueOf(hashNWS(ct.request));
184    return ct;
185  }
186
187  private String getNameForSystem(String system) {
188    if (system.equals("http://snomed.info/sct"))
189      return "snomed";
190    if (system.equals("http://www.nlm.nih.gov/research/umls/rxnorm"))
191      return "rxnorm";
192    if (system.equals("http://loinc.org"))
193      return "loinc";
194    if (system.equals("http://unitsofmeasure.org"))
195      return "ucum";
196    if (system.startsWith("http://hl7.org/fhir/sid/"))
197      return system.substring(24).replace("/", "");
198    if (system.startsWith("urn:iso:std:iso:"))
199      return "iso"+system.substring(16).replace(":", "");
200    if (system.startsWith("http://terminology.hl7.org/CodeSystem/"))
201      return system.substring(38).replace("/", "");
202    if (system.startsWith("http://hl7.org/fhir/"))
203      return system.substring(20).replace("/", "");
204    if (system.equals("urn:ietf:bcp:47"))
205      return "lang";
206    if (system.equals("urn:ietf:bcp:13"))
207      return "mimetypes";
208    if (system.equals("urn:iso:std:iso:11073:10101"))
209      return "11073";
210    if (system.equals("http://dicom.nema.org/resources/ontology/DCM"))
211      return "dicom";
212    return system.replace("/", "_").replace(":", "_");
213  }
214
215  public NamedCache getNamedCache(CacheToken cacheToken) {
216    NamedCache nc = caches.get(cacheToken.name);
217    if (nc == null) {
218      nc = new NamedCache();
219      nc.name = cacheToken.name;
220      caches.put(nc.name, nc);
221    }
222    return nc;
223  }
224  
225  public ValueSetExpansionOutcome getExpansion(CacheToken cacheToken) {
226    synchronized (lock) {
227      NamedCache nc = getNamedCache(cacheToken);
228      CacheEntry e = nc.map.get(cacheToken.key);
229      if (e == null)
230        return null;
231      else
232        return e.e;
233    }
234  }
235
236  public void cacheExpansion(CacheToken cacheToken, ValueSetExpansionOutcome res, boolean persistent) {
237    synchronized (lock) {      
238      NamedCache nc = getNamedCache(cacheToken);
239      CacheEntry e = new CacheEntry();
240      e.request = cacheToken.request;
241      e.persistent = persistent;
242      e.e = res;
243      store(cacheToken, persistent, nc, e);
244    }    
245  }
246
247  public void store(CacheToken cacheToken, boolean persistent, NamedCache nc, CacheEntry e) {
248    boolean n = nc.map.containsKey(cacheToken.key);
249    nc.map.put(cacheToken.key, e);
250    if (persistent) {
251      if (n) {
252        for (int i = nc.list.size()- 1; i>= 0; i--) {
253          if (nc.list.get(i).request.equals(e.request)) {
254            nc.list.remove(i);
255          }
256        }
257      }
258      nc.list.add(e);
259      save(nc);  
260    }
261  }
262
263  public ValidationResult getValidation(CacheToken cacheToken) {
264    synchronized (lock) {
265      NamedCache nc = getNamedCache(cacheToken);
266      CacheEntry e = nc.map.get(cacheToken.key);
267      if (e == null)
268        return null;
269      else
270        return e.v;
271    }
272  }
273
274  public void cacheValidation(CacheToken cacheToken, ValidationResult res, boolean persistent) {
275    synchronized (lock) {      
276      NamedCache nc = getNamedCache(cacheToken);
277      CacheEntry e = new CacheEntry();
278      e.request = cacheToken.request;
279      e.persistent = persistent;
280      e.v = res;
281      store(cacheToken, persistent, nc, e);
282    }    
283  }
284
285  
286  // persistence
287  
288  public void save() {
289    
290  }
291  
292  private void save(NamedCache nc) {
293    if (folder == null)
294      return;
295    
296    try {
297      OutputStreamWriter sw = new OutputStreamWriter(new FileOutputStream(Utilities.path(folder, nc.name+".cache")), "UTF-8");
298      sw.write(ENTRY_MARKER+"\r\n");
299      JsonParser json = new JsonParser();
300      json.setOutputStyle(OutputStyle.PRETTY);
301      for (CacheEntry ce : nc.list) {
302        sw.write(ce.request.trim());
303        sw.write(BREAK+"\r\n");
304        if (ce.e != null) {
305          sw.write("e: {\r\n");
306          if (ce.e.getValueset() != null)
307            sw.write("  \"valueSet\" : "+json.composeString(ce.e.getValueset()).trim()+",\r\n");
308          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.e.getError()).trim()+"\"\r\n}\r\n");
309        } else {
310          sw.write("v: {\r\n");
311          sw.write("  \"display\" : \""+Utilities.escapeJson(ce.v.getDisplay()).trim()+"\",\r\n");
312          sw.write("  \"severity\" : "+(ce.v.getSeverity() == null ? "null" : "\""+ce.v.getSeverity().toCode().trim()+"\"")+",\r\n");
313          sw.write("  \"error\" : \""+Utilities.escapeJson(ce.v.getMessage()).trim()+"\"\r\n}\r\n");
314        }
315        sw.write(ENTRY_MARKER+"\r\n");
316      }      
317      sw.close();
318    } catch (Exception e) {
319      System.out.println("error saving "+nc.name+": "+e.getMessage());
320    }
321  }
322
323  private void load() throws FHIRException {
324    for (String fn : new File(folder).list()) {
325      if (fn.endsWith(".cache") && !fn.equals("validation.cache")) {
326        try {
327          String title = fn.substring(0, fn.lastIndexOf("."));
328          NamedCache nc = new NamedCache();
329          nc.name = title;
330          caches.put(title, nc);
331          String src = TextFile.fileToString(Utilities.path(folder, fn));
332          if (src.startsWith("?"))
333            src = src.substring(1);
334          int i = src.indexOf(ENTRY_MARKER); 
335          while (i > -1) {
336            String s = src.substring(0, i);
337            src = src.substring(i+ENTRY_MARKER.length()+1);
338            i = src.indexOf(ENTRY_MARKER);
339            if (!Utilities.noString(s)) {
340              int j = s.indexOf(BREAK);
341              String q = s.substring(0, j);
342              String p = s.substring(j+BREAK.length()+1).trim();
343              CacheEntry ce = new CacheEntry();
344              ce.persistent = true;
345              ce.request = q;
346              boolean e = p.charAt(0) == 'e';
347              p = p.substring(3);
348              JsonObject o = (JsonObject) new com.google.gson.JsonParser().parse(p);
349              String error = loadJS(o.get("error"));
350              if (e) {
351                if (o.has("valueSet"))
352                  ce.e = new ValueSetExpansionOutcome((ValueSet) new JsonParser().parse(o.getAsJsonObject("valueSet")), error, TerminologyServiceErrorClass.UNKNOWN);
353                else
354                  ce.e = new ValueSetExpansionOutcome(error, TerminologyServiceErrorClass.UNKNOWN);
355              } else {
356                IssueSeverity severity = o.get("severity") instanceof JsonNull ? null :  IssueSeverity.fromCode(o.get("severity").getAsString());
357                String display = loadJS(o.get("display"));
358                ce.v = new ValidationResult(severity, error, new ConceptDefinitionComponent().setDisplay(display));
359              }
360              nc.map.put(String.valueOf(hashNWS(ce.request)), ce);
361              nc.list.add(ce);
362            }
363          }        
364        } catch (Exception e) {
365          throw new FHIRException("Error loading "+fn+": "+e.getMessage(), e);
366        }
367      }
368    }
369  }
370  
371  private String loadJS(JsonElement e) {
372    if (e == null)
373      return null;
374    if (!(e instanceof JsonPrimitive))
375      return null;
376    String s = e.getAsString();
377    if ("".equals(s))
378      return null;
379    return s;
380  }
381
382  private String hashNWS(String s) {
383    return String.valueOf(s.replace("\r", "").replace("\n", "").replace(" ", "").hashCode());
384  }
385
386  // management
387  
388  public TerminologyCache copy() {
389    // TODO Auto-generated method stub
390    return null;
391  }
392  
393  public String summary(ValueSet vs) {
394    if (vs == null)
395      return "null";
396    
397    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
398    for (ConceptSetComponent cc : vs.getCompose().getInclude())
399      b.append("Include "+getIncSummary(cc));
400    for (ConceptSetComponent cc : vs.getCompose().getExclude())
401      b.append("Exclude "+getIncSummary(cc));
402    return b.toString();
403  }
404
405  private String getIncSummary(ConceptSetComponent cc) {
406    CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
407    for (UriType vs : cc.getValueSet())
408      b.append(vs.asStringValue());
409    String vsd = b.length() > 0 ? " where the codes are in the value sets ("+b.toString()+")" : "";
410    String system = cc.getSystem();
411    if (cc.hasConcept())
412      return Integer.toString(cc.getConcept().size())+" codes from "+system+vsd;
413    if (cc.hasFilter()) {
414      String s = "";
415      for (ConceptSetFilterComponent f : cc.getFilter()) {
416        if (!Utilities.noString(s))
417          s = s + " & ";
418        s = s + f.getProperty()+" "+(f.hasOp() ? f.getOp().toCode() : "?")+" "+f.getValue();
419      }
420      return "from "+system+" where "+s+vsd;
421    }
422    return "All codes from "+system+vsd;
423  }
424
425  public String summary(Coding code) {
426    return code.getSystem()+"#"+code.getCode()+": \""+code.getDisplay()+"\"";
427  }
428
429
430  public String summary(CodeableConcept code) {
431    StringBuilder b = new StringBuilder();
432    b.append("{");
433    boolean first = true;
434    for (Coding c : code.getCoding()) {
435      if (first) first = false; else b.append(",");
436      b.append(summary(c));
437    }
438    b.append("}: \"");
439    b.append(code.getText());
440    b.append("\"");
441    return b.toString();
442  }
443  
444}