001package org.hl7.fhir.r5.context;
002
003/*-
004 * #%L
005 * org.hl7.fhir.r5
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023import java.io.FileNotFoundException;
024import java.io.IOException;
025import java.util.ArrayList;
026import java.util.Collections;
027import java.util.Comparator;
028import java.util.Date;
029import java.util.EnumSet;
030import java.util.HashMap;
031import java.util.HashSet;
032import java.util.List;
033import java.util.Map;
034import java.util.Set;
035
036import org.apache.commons.lang3.StringUtils;
037import org.fhir.ucum.UcumService;
038import org.hl7.fhir.exceptions.DefinitionException;
039import org.hl7.fhir.exceptions.FHIRException;
040import org.hl7.fhir.exceptions.TerminologyServiceException;
041import org.hl7.fhir.r5.conformance.ProfileUtilities;
042import org.hl7.fhir.r5.context.BaseWorkerContext.MetadataResourceVersionComparator;
043import org.hl7.fhir.r5.context.IWorkerContext.ILoggingService.LogCategory;
044import org.hl7.fhir.r5.context.TerminologyCache.CacheToken;
045import org.hl7.fhir.r5.model.BooleanType;
046import org.hl7.fhir.r5.model.CapabilityStatement;
047import org.hl7.fhir.r5.model.CodeSystem;
048import org.hl7.fhir.r5.model.CodeSystem.CodeSystemContentMode;
049import org.hl7.fhir.r5.model.CodeSystem.ConceptDefinitionComponent;
050import org.hl7.fhir.r5.model.CodeableConcept;
051import org.hl7.fhir.r5.model.Coding;
052import org.hl7.fhir.r5.model.ConceptMap;
053import org.hl7.fhir.r5.model.Constants;
054import org.hl7.fhir.r5.model.ElementDefinition.ElementDefinitionBindingComponent;
055import org.hl7.fhir.r5.model.ImplementationGuide;
056import org.hl7.fhir.r5.model.CanonicalResource;
057import org.hl7.fhir.r5.model.NamingSystem;
058import org.hl7.fhir.r5.model.NamingSystem.NamingSystemIdentifierType;
059import org.hl7.fhir.r5.model.NamingSystem.NamingSystemUniqueIdComponent;
060import org.hl7.fhir.r5.model.OperationDefinition;
061import org.hl7.fhir.r5.model.Parameters;
062import org.hl7.fhir.r5.model.Parameters.ParametersParameterComponent;
063import org.hl7.fhir.r5.model.PlanDefinition;
064import org.hl7.fhir.r5.model.Questionnaire;
065import org.hl7.fhir.r5.model.Reference;
066import org.hl7.fhir.r5.model.Resource;
067import org.hl7.fhir.r5.model.SearchParameter;
068import org.hl7.fhir.r5.model.StringType;
069import org.hl7.fhir.r5.model.StructureDefinition;
070import org.hl7.fhir.r5.model.StructureMap;
071import org.hl7.fhir.r5.model.TerminologyCapabilities;
072import org.hl7.fhir.r5.model.TerminologyCapabilities.TerminologyCapabilitiesCodeSystemComponent;
073import org.hl7.fhir.r5.model.ValueSet;
074import org.hl7.fhir.r5.model.ValueSet.ConceptSetComponent;
075import org.hl7.fhir.r5.model.ValueSet.ValueSetComposeComponent;
076import org.hl7.fhir.r5.terminologies.TerminologyClient;
077import org.hl7.fhir.r5.terminologies.ValueSetCheckerSimple;
078import org.hl7.fhir.r5.terminologies.ValueSetExpander.TerminologyServiceErrorClass;
079import org.hl7.fhir.r5.terminologies.ValueSetExpander.ValueSetExpansionOutcome;
080import org.hl7.fhir.r5.terminologies.ValueSetExpanderSimple;
081import org.hl7.fhir.r5.utils.ToolingExtensions;
082import org.hl7.fhir.r5.utils.client.ToolingClientLogger;
083import org.hl7.fhir.utilities.OIDUtils;
084import org.hl7.fhir.utilities.TerminologyServiceOptions;
085import org.hl7.fhir.utilities.TranslationServices;
086import org.hl7.fhir.utilities.Utilities;
087import org.hl7.fhir.utilities.VersionUtilities;
088import org.hl7.fhir.utilities.validation.ValidationOptions;
089import org.hl7.fhir.utilities.validation.ValidationOptions.ValueSetMode;
090import org.hl7.fhir.utilities.validation.ValidationMessage.IssueSeverity;
091import org.hl7.fhir.utilities.validation.ValidationMessage.IssueType;
092
093import com.google.gson.JsonObject;
094
095public abstract class BaseWorkerContext implements IWorkerContext {
096
097  public class MetadataResourceVersionComparator<T extends CanonicalResource> implements Comparator<T> {
098
099    private List<T> list;
100
101    public MetadataResourceVersionComparator(List<T> list) {
102      this.list = list;
103    }
104
105    @Override
106    public int compare(T arg1, T arg2) {
107      String v1 = arg1.getVersion();
108      String v2 = arg2.getVersion();
109      if (v1 == null && v2 == null) {
110        return Integer.compare(list.indexOf(arg1), list.indexOf(arg2)); // retain original order
111      } else if (v1 == null) {
112        return -1;
113      } else if (v2 == null) {
114        return 1;
115      } else {
116        String mm1 = VersionUtilities.getMajMin(v1);
117        String mm2 = VersionUtilities.getMajMin(v2);
118        if (mm1 == null || mm2 == null) {
119          return v1.compareTo(v2);
120        } else {
121          return mm1.compareTo(mm2);
122        }
123      }
124    }
125  }
126
127  private Object lock = new Object(); // used as a lock for the data that follows
128  
129  private Map<String, Map<String, Resource>> allResourcesById = new HashMap<String, Map<String, Resource>>();
130  // all maps are to the full URI
131  private CanonicalResourceManager<CodeSystem> codeSystems = new CanonicalResourceManager<CodeSystem>(false);
132  private Set<String> supportedCodeSystems = new HashSet<String>();
133  private CanonicalResourceManager<ValueSet> valueSets = new CanonicalResourceManager<ValueSet>(false);
134  private CanonicalResourceManager<ConceptMap> maps = new CanonicalResourceManager<ConceptMap>(false);
135  protected CanonicalResourceManager<StructureMap> transforms = new CanonicalResourceManager<StructureMap>(false);
136  private CanonicalResourceManager<StructureDefinition> structures = new CanonicalResourceManager<StructureDefinition>(false);
137  private CanonicalResourceManager<ImplementationGuide> guides = new CanonicalResourceManager<ImplementationGuide>(false);
138  private CanonicalResourceManager<CapabilityStatement> capstmts = new CanonicalResourceManager<CapabilityStatement>(false);
139  private CanonicalResourceManager<SearchParameter> searchParameters = new CanonicalResourceManager<SearchParameter>(false);
140  private CanonicalResourceManager<Questionnaire> questionnaires = new CanonicalResourceManager<Questionnaire>(false);
141  private CanonicalResourceManager<OperationDefinition> operations = new CanonicalResourceManager<OperationDefinition>(false);
142  private CanonicalResourceManager<PlanDefinition> plans = new CanonicalResourceManager<PlanDefinition>(false);
143  private List<NamingSystem> systems = new ArrayList<NamingSystem>();
144  private UcumService ucumService;
145  protected Map<String, byte[]> binaries = new HashMap<String, byte[]>();
146
147  protected Map<String, Map<String, ValidationResult>> validationCache = new HashMap<String, Map<String,ValidationResult>>();
148  protected String tsServer;
149  protected String name;
150  private boolean allowLoadingDuplicates;
151
152  protected TerminologyClient txClient;
153  protected ToolingClientLogger txLog;
154  private TerminologyCapabilities txcaps;
155  private boolean canRunWithoutTerminology;
156  protected boolean noTerminologyServer;
157  private int expandCodesLimit = 1000;
158  protected ILoggingService logger;
159  protected Parameters expParameters;
160  private TranslationServices translator = new NullTranslator();
161  protected TerminologyCache txCache;
162
163  private boolean tlogging = true;
164  
165  public BaseWorkerContext() throws FileNotFoundException, IOException, FHIRException {
166    super();
167    txCache = new TerminologyCache(lock, null);
168  }
169
170  public BaseWorkerContext(CanonicalResourceManager<CodeSystem> codeSystems, CanonicalResourceManager<ValueSet> valueSets, CanonicalResourceManager<ConceptMap> maps, CanonicalResourceManager<StructureDefinition> profiles, 
171      CanonicalResourceManager<ImplementationGuide> guides) throws FileNotFoundException, IOException, FHIRException {
172    super();
173    this.codeSystems = codeSystems;
174    this.valueSets = valueSets;
175    this.maps = maps;
176    this.structures = profiles;
177    this.guides = guides;
178    txCache = new TerminologyCache(lock, null);
179  }
180
181  protected void copy(BaseWorkerContext other) {
182    synchronized (other.lock) { // tricky, because you need to lock this as well, but it's really not in use yet 
183      allResourcesById.putAll(other.allResourcesById);
184      translator = other.translator;
185      codeSystems.copy(other.codeSystems);
186      txcaps = other.txcaps;
187      valueSets.copy(other.valueSets);
188      maps.copy(other.maps);
189      transforms.copy(other.transforms);
190      structures.copy(other.structures);
191      searchParameters.copy(other.searchParameters);
192      plans.copy(other.plans);
193      questionnaires.copy(other.questionnaires);
194      operations.copy(other.operations);
195      systems.addAll(other.systems);
196      guides.copy(other.guides);
197      capstmts.copy(other.capstmts);
198
199      allowLoadingDuplicates = other.allowLoadingDuplicates;
200      tsServer = other.tsServer;
201      name = other.name;
202      txClient = other.txClient;
203      txLog = other.txLog;
204      txcaps = other.txcaps;
205      canRunWithoutTerminology = other.canRunWithoutTerminology;
206      noTerminologyServer = other.noTerminologyServer;
207      if (other.txCache != null)
208        txCache = other.txCache.copy();
209      expandCodesLimit = other.expandCodesLimit;
210      logger = other.logger;
211      expParameters = other.expParameters;
212    }
213  }
214  
215  public void cacheResource(Resource r) throws FHIRException {
216    synchronized (lock) {
217      Map<String, Resource> map = allResourcesById.get(r.fhirType());
218      if (map == null) {
219        map = new HashMap<String, Resource>();
220        allResourcesById.put(r.fhirType(), map);
221      }
222      map.put(r.getId(), r);
223
224      if (r instanceof CanonicalResource) {
225        CanonicalResource m = (CanonicalResource) r;
226        String url = m.getUrl();
227        if (!allowLoadingDuplicates && hasResource(r.getClass(), url)) {
228          // spcial workaround for known problems with existing packages
229          if (Utilities.existsInList(url, "http://hl7.org/fhir/SearchParameter/example")) {
230            return;
231          }
232          throw new DefinitionException("Duplicate Resource " + url);
233        }
234        if (r instanceof StructureDefinition)
235          structures.see((StructureDefinition) m);
236        else if (r instanceof ValueSet)
237          valueSets.see((ValueSet) m);
238        else if (r instanceof CodeSystem)
239          codeSystems.see((CodeSystem) m);
240        else if (r instanceof ImplementationGuide)
241          guides.see((ImplementationGuide) m);
242        else if (r instanceof CapabilityStatement)
243          capstmts.see((CapabilityStatement) m);
244        else if (r instanceof SearchParameter)
245          searchParameters.see((SearchParameter) m);
246        else if (r instanceof PlanDefinition)
247          plans.see((PlanDefinition) m);
248        else if (r instanceof OperationDefinition)
249          operations.see((OperationDefinition) m);
250        else if (r instanceof Questionnaire)
251          questionnaires.see((Questionnaire) m);
252        else if (r instanceof ConceptMap)
253          maps.see((ConceptMap) m);
254        else if (r instanceof StructureMap)
255          transforms.see((StructureMap) m);
256        else if (r instanceof NamingSystem)
257          systems.add((NamingSystem) r);
258      }
259    }
260  }
261
262  /*
263   *  Compare business versions, returning "true" if the candidate newer version is in fact newer than the oldVersion
264   *  Comparison will work for strictly numeric versions as well as multi-level versions separated by ., -, _, : or space
265   *  Failing that, it will do unicode-based character ordering.
266   *  E.g. 1.5.3 < 1.14.3
267   *       2017-3-10 < 2017-12-7
268   *       A3 < T2
269   */
270  private boolean laterVersion(String newVersion, String oldVersion) {
271    // Compare business versions, retur
272    newVersion = newVersion.trim();
273    oldVersion = oldVersion.trim();
274    if (StringUtils.isNumeric(newVersion) && StringUtils.isNumeric(oldVersion))
275      return Double.parseDouble(newVersion) > Double.parseDouble(oldVersion);
276    else if (hasDelimiter(newVersion, oldVersion, "."))
277      return laterDelimitedVersion(newVersion, oldVersion, "\\.");
278    else if (hasDelimiter(newVersion, oldVersion, "-"))
279      return laterDelimitedVersion(newVersion, oldVersion, "\\-");
280    else if (hasDelimiter(newVersion, oldVersion, "_"))
281      return laterDelimitedVersion(newVersion, oldVersion, "\\_");
282    else if (hasDelimiter(newVersion, oldVersion, ":"))
283      return laterDelimitedVersion(newVersion, oldVersion, "\\:");
284    else if (hasDelimiter(newVersion, oldVersion, " "))
285      return laterDelimitedVersion(newVersion, oldVersion, "\\ ");
286    else {
287      return newVersion.compareTo(oldVersion) > 0;
288    }
289  }
290  
291  /*
292   * Returns true if both strings include the delimiter and have the same number of occurrences of it
293   */
294  private boolean hasDelimiter(String s1, String s2, String delimiter) {
295    return s1.contains(delimiter) && s2.contains(delimiter) && s1.split(delimiter).length == s2.split(delimiter).length;
296  }
297
298  private boolean laterDelimitedVersion(String newVersion, String oldVersion, String delimiter) {
299    String[] newParts = newVersion.split(delimiter);
300    String[] oldParts = oldVersion.split(delimiter);
301    for (int i = 0; i < newParts.length; i++) {
302      if (!newParts[i].equals(oldParts[i]))
303        return laterVersion(newParts[i], oldParts[i]);
304    }
305    // This should never happen
306    throw new Error("Delimited versions have exact match for delimiter '"+delimiter+"' : "+newParts+" vs "+oldParts);
307  }
308  
309  protected <T extends CanonicalResource> void seeMetadataResource(T r, Map<String, T> map, List<T> list, boolean addId) throws FHIRException {
310//    if (addId)
311    //      map.put(r.getId(), r); // todo: why?
312    list.add(r);
313    if (r.hasUrl()) {
314      // first, this is the correct reosurce for this version (if it has a version)
315      if (r.hasVersion()) {
316        map.put(r.getUrl()+"|"+r.getVersion(), r);
317      }
318      // if we haven't get anything for this url, it's the correct version
319      if (!map.containsKey(r.getUrl()))
320        map.put(r.getUrl(), r);
321      else {
322        List<T> rl = new ArrayList<T>();
323        for (T t : list) {
324          if (t.getUrl().equals(r.getUrl()) && !rl.contains(t)) {
325            rl.add(t);
326          }
327        }
328        Collections.sort(rl, new MetadataResourceVersionComparator<T>(list));
329        map.put(r.getUrl(), rl.get(rl.size()-1));
330        T latest = null;
331        for (T t : rl) {
332          if (VersionUtilities.versionsCompatible(t.getVersion(), r.getVersion())) {
333            latest = t;
334          }
335        }
336        if (latest != null) { // might be null if it's not using semver
337          map.put(r.getUrl()+"|"+VersionUtilities.getMajMin(latest.getVersion()), rl.get(rl.size()-1));
338        }
339      }
340    }
341  }  
342
343  @Override
344  public CodeSystem fetchCodeSystem(String system) {
345    synchronized (lock) {
346      return codeSystems.get(system);
347    }
348  } 
349
350  @Override
351  public boolean supportsSystem(String system) throws TerminologyServiceException {
352    synchronized (lock) {
353      if (codeSystems.has(system) && codeSystems.get(system).getContent() != CodeSystemContentMode.NOTPRESENT)
354        return true;
355      else if (supportedCodeSystems.contains(system))
356        return true;
357      else if (system.startsWith("http://example.org") || system.startsWith("http://acme.com") || system.startsWith("http://hl7.org/fhir/valueset-") || system.startsWith("urn:oid:"))
358        return false;
359      else {
360        if (noTerminologyServer)
361          return false;
362        if (txcaps == null) {
363          try {
364            log("Terminology server: Check for supported code systems for "+system);
365            txcaps = txClient.getTerminologyCapabilities();
366          } catch (Exception e) {
367            if (canRunWithoutTerminology) {
368              noTerminologyServer = true;
369              log("==============!! Running without terminology server !! ==============");
370              if (txClient!=null) {
371                log("txServer = "+txClient.getAddress());
372                log("Error = "+e.getMessage()+"");
373              }
374              log("=====================================================================");
375              return false;
376            } else
377              throw new TerminologyServiceException(e);
378          }
379          if (txcaps != null) {
380            for (TerminologyCapabilitiesCodeSystemComponent tccs : txcaps.getCodeSystem()) {
381              supportedCodeSystems.add(tccs.getUri());
382            }
383          }
384          if (supportedCodeSystems.contains(system))
385            return true;
386        }
387      }
388      return false;
389    }
390  }
391
392  private void log(String message) {
393    if (logger != null)
394      logger.logMessage(message);
395    else
396      System.out.println(message);
397  }
398
399
400  protected void tlog(String msg) {
401    if (tlogging )
402      if (logger != null)
403        logger.logDebugMessage(LogCategory.TX, msg);
404      else 
405        System.out.println("-tx: "+msg);
406  }
407
408  // --- expansion support ------------------------------------------------------------------------------------------------------------
409
410  public int getExpandCodesLimit() {
411    return expandCodesLimit;
412  }
413
414  public void setExpandCodesLimit(int expandCodesLimit) {
415    this.expandCodesLimit = expandCodesLimit;
416  }
417
418  @Override
419  public ValueSetExpansionOutcome expandVS(ElementDefinitionBindingComponent binding, boolean cacheOk, boolean heirarchical) throws FHIRException {
420    ValueSet vs = null;
421    vs = fetchResource(ValueSet.class, binding.getValueSet());
422    if (vs == null)
423      throw new FHIRException("Unable to resolve value Set "+binding.getValueSet());
424    return expandVS(vs, cacheOk, heirarchical);
425  }
426  
427  @Override
428  public ValueSetExpansionOutcome expandVS(ConceptSetComponent inc, boolean hierarchical) throws TerminologyServiceException {
429    ValueSet vs = new ValueSet();
430    vs.setCompose(new ValueSetComposeComponent());
431    vs.getCompose().getInclude().add(inc);
432    CacheToken cacheToken = txCache.generateExpandToken(vs, hierarchical);
433    ValueSetExpansionOutcome res;
434    res = txCache.getExpansion(cacheToken);
435    if (res != null)
436      return res;
437    Parameters p = expParameters.copy(); 
438    p.setParameter("includeDefinition", false);
439    p.setParameter("excludeNested", !hierarchical);
440    
441    if (noTerminologyServer)
442      return new ValueSetExpansionOutcome("Error expanding ValueSet: running without terminology services", TerminologyServiceErrorClass.NOSERVICE);
443    Map<String, String> params = new HashMap<String, String>();
444    params.put("_limit", Integer.toString(expandCodesLimit ));
445    params.put("_incomplete", "true");
446    tlog("$expand on "+txCache.summary(vs));
447    try {
448      ValueSet result = txClient.expandValueset(vs, p, params);
449      res = new ValueSetExpansionOutcome(result).setTxLink(txLog.getLastId());  
450    } catch (Exception e) {
451      res = new ValueSetExpansionOutcome(e.getMessage() == null ? e.getClass().getName() : e.getMessage(), TerminologyServiceErrorClass.UNKNOWN);
452      if (txLog != null)
453        res.setTxLink(txLog.getLastId());
454    }
455    txCache.cacheExpansion(cacheToken, res, TerminologyCache.PERMANENT);
456    return res;
457  }
458
459  @Override
460  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical) {
461    if (expParameters == null)
462      throw new Error("No Expansion Parameters provided");
463    Parameters p = expParameters.copy(); 
464    return expandVS(vs, cacheOk, heirarchical, p);
465  }
466  
467  public ValueSetExpansionOutcome expandVS(ValueSet vs, boolean cacheOk, boolean heirarchical, Parameters p)  {
468    if (p == null)
469      throw new Error("No Parameters provided to expandVS");
470    if (vs.hasExpansion()) {
471      return new ValueSetExpansionOutcome(vs.copy());
472    }
473    if (!vs.hasUrl())
474      throw new Error("no value set");
475    
476      CacheToken cacheToken = txCache.generateExpandToken(vs, heirarchical);
477      ValueSetExpansionOutcome res;
478      if (cacheOk) {
479        res = txCache.getExpansion(cacheToken);
480        if (res != null)
481          return res;
482      }
483      p.setParameter("includeDefinition", false);
484      p.setParameter("excludeNested", !heirarchical);
485      
486      // ok, first we try to expand locally
487      try {
488        ValueSetExpanderSimple vse = new ValueSetExpanderSimple(this);
489        res = vse.doExpand(vs, p);
490        if (!res.getValueset().hasUrl())
491          throw new Error("no url in expand value set");
492        txCache.cacheExpansion(cacheToken, res, TerminologyCache.TRANSIENT);
493        return res;
494      } catch (Exception e) {
495      }
496      
497      // if that failed, we try to expand on the server
498      if (noTerminologyServer)
499        return new ValueSetExpansionOutcome("Error expanding ValueSet: running without terminology services", TerminologyServiceErrorClass.NOSERVICE);
500      Map<String, String> params = new HashMap<String, String>();
501      params.put("_limit", Integer.toString(expandCodesLimit ));
502      params.put("_incomplete", "true");
503      tlog("$expand on "+txCache.summary(vs));
504      try {
505        ValueSet result = txClient.expandValueset(vs, p, params);
506        if (!result.hasUrl())
507          result.setUrl(vs.getUrl());
508        if (!result.hasUrl())
509          throw new Error("no url in expand value set 2");
510        res = new ValueSetExpansionOutcome(result).setTxLink(txLog.getLastId());  
511      } catch (Exception e) {
512        res = new ValueSetExpansionOutcome(e.getMessage() == null ? e.getClass().getName() : e.getMessage(), TerminologyServiceErrorClass.UNKNOWN).setTxLink(txLog == null ? null : txLog.getLastId());
513      }
514      txCache.cacheExpansion(cacheToken, res, TerminologyCache.PERMANENT);
515      return res;
516  }
517
518
519  private boolean hasTooCostlyExpansion(ValueSet valueset) {
520    return valueset != null && valueset.hasExpansion() && ToolingExtensions.hasExtension(valueset.getExpansion(), "http://hl7.org/fhir/StructureDefinition/valueset-toocostly");
521  }
522  // --- validate code -------------------------------------------------------------------------------
523  
524  @Override
525  public ValidationResult validateCode(ValidationOptions options, String system, String code, String display) {
526    assert options != null;
527    Coding c = new Coding(system, code, display);
528    return validateCode(options, c, null);
529  }
530
531  @Override
532  public ValidationResult validateCode(ValidationOptions options, String system, String code, String display, ValueSet vs) {
533    assert options != null;
534    Coding c = new Coding(system, code, display);
535    return validateCode(options, c, vs);
536  }
537
538  @Override
539  public ValidationResult validateCode(ValidationOptions options, String code, ValueSet vs) {
540    assert options != null;
541    Coding c = new Coding(null, code, null);
542    return validateCode(options.guessSystem(), c, vs);
543  }
544
545  @Override
546  public ValidationResult validateCode(ValidationOptions options, Coding code, ValueSet vs) {
547    assert options != null;
548    
549    CacheToken cacheToken = txCache != null ? txCache.generateValidationToken(options, code, vs) : null;
550    ValidationResult res = null;
551    if (txCache != null) 
552      res = txCache.getValidation(cacheToken);
553    if (res != null)
554      return res;
555
556    if (options.isUseClient()) {
557      // ok, first we try to validate locally
558      try {
559        ValueSetCheckerSimple vsc = new ValueSetCheckerSimple(options, vs, this); 
560        res = vsc.validateCode(code);
561        if (txCache != null)
562          txCache.cacheValidation(cacheToken, res, TerminologyCache.TRANSIENT);
563        return res;
564      } catch (Exception e) {
565      }
566    }
567    
568    if (!options.isUseServer()) {
569      return new ValidationResult(IssueSeverity.WARNING, "Unable to validate code without using server", TerminologyServiceErrorClass.BLOCKED_BY_OPTIONS);      
570    }
571    
572    // if that failed, we try to validate on the server
573    if (noTerminologyServer) {
574      return new ValidationResult(IssueSeverity.ERROR,  "Error validating code: running without terminology services", TerminologyServiceErrorClass.NOSERVICE);
575    }
576    String csumm =  txCache != null ? txCache.summary(code) : null;
577    if (txCache != null) {
578      tlog("$validate "+csumm+" for "+ txCache.summary(vs));
579    } else {
580      tlog("$validate "+csumm+" before cache exists");
581    }
582    try {
583      Parameters pIn = new Parameters();
584      pIn.addParameter().setName("coding").setValue(code);
585      if (options.isGuessSystem())
586        pIn.addParameter().setName("implySystem").setValue(new BooleanType(true));
587      setTerminologyOptions(options, pIn);
588      res = validateOnServer(vs, pIn);
589    } catch (Exception e) {
590      res = new ValidationResult(IssueSeverity.ERROR, e.getMessage() == null ? e.getClass().getName() : e.getMessage()).setTxLink(txLog == null ? null : txLog.getLastId());
591    }
592    if (txCache != null)
593      txCache.cacheValidation(cacheToken, res, TerminologyCache.PERMANENT);
594    return res;
595  }
596
597  private void setTerminologyOptions(ValidationOptions options, Parameters pIn) {
598    if (!Utilities.noString(options.getLanguage())) {
599      pIn.addParameter("displayLanguage", options.getLanguage());
600    }
601    if (options.getValueSetMode() != ValueSetMode.ALL_CHECKS) {
602      pIn.addParameter("valueSetMode", options.getValueSetMode().toString());
603    }
604  }
605
606  @Override
607  public ValidationResult validateCode(ValidationOptions options, CodeableConcept code, ValueSet vs) {
608    CacheToken cacheToken = txCache.generateValidationToken(options, code, vs);
609    ValidationResult res = txCache.getValidation(cacheToken);
610    if (res != null)
611      return res;
612
613    if (options.isUseClient()) {
614      // ok, first we try to validate locally
615      try {
616        ValueSetCheckerSimple vsc = new ValueSetCheckerSimple(options, vs, this); 
617        res = vsc.validateCode(code);
618        txCache.cacheValidation(cacheToken, res, TerminologyCache.TRANSIENT);
619        return res;
620      } catch (Exception e) {
621      }
622    }
623
624    if (!options.isUseServer()) {
625      return new ValidationResult(IssueSeverity.WARNING, "Unable to validate code without using server", TerminologyServiceErrorClass.BLOCKED_BY_OPTIONS);      
626    }
627    
628    // if that failed, we try to validate on the server
629    if (noTerminologyServer)
630      return new ValidationResult(IssueSeverity.ERROR, "Error validating code: running without terminology services", TerminologyServiceErrorClass.NOSERVICE);
631    tlog("$validate "+txCache.summary(code)+" for "+ txCache.summary(vs));
632    try {
633      Parameters pIn = new Parameters();
634      pIn.addParameter().setName("codeableConcept").setValue(code);
635      setTerminologyOptions(options, pIn);
636      res = validateOnServer(vs, pIn);
637    } catch (Exception e) {
638      res = new ValidationResult(IssueSeverity.ERROR, e.getMessage() == null ? e.getClass().getName() : e.getMessage()).setTxLink(txLog.getLastId());
639    }
640    txCache.cacheValidation(cacheToken, res, TerminologyCache.PERMANENT);
641    return res;
642  }
643
644  private ValidationResult validateOnServer(ValueSet vs, Parameters pin) throws FHIRException {
645    if (vs != null)
646      pin.addParameter().setName("valueSet").setResource(vs);
647    for (ParametersParameterComponent pp : pin.getParameter())
648      if (pp.getName().equals("profile"))
649        throw new Error("Can only specify profile in the context");
650    if (expParameters == null)
651      throw new Error("No ExpansionProfile provided");
652    pin.addParameter().setName("profile").setResource(expParameters);
653    txLog.clearLastId();
654    Parameters pOut;
655    if (vs == null)
656      pOut = txClient.validateCS(pin);
657    else
658      pOut = txClient.validateVS(pin);
659    boolean ok = false;
660    String message = "No Message returned";
661    String display = null;
662    TerminologyServiceErrorClass err = TerminologyServiceErrorClass.UNKNOWN;
663    for (ParametersParameterComponent p : pOut.getParameter()) {
664      if (p.getName().equals("result"))
665        ok = ((BooleanType) p.getValue()).getValue().booleanValue();
666      else if (p.getName().equals("message"))
667        message = ((StringType) p.getValue()).getValue();
668      else if (p.getName().equals("display"))
669        display = ((StringType) p.getValue()).getValue();
670      else if (p.getName().equals("cause")) {
671        try {
672          IssueType it = IssueType.fromCode(((StringType) p.getValue()).getValue());
673          if (it == IssueType.UNKNOWN)
674            err = TerminologyServiceErrorClass.UNKNOWN;
675          else if (it == IssueType.NOTSUPPORTED)
676            err = TerminologyServiceErrorClass.VALUESET_UNSUPPORTED;
677        } catch (FHIRException e) {
678        }
679      }
680    }
681    if (!ok)
682      return new ValidationResult(IssueSeverity.ERROR, message, err).setTxLink(txLog.getLastId()).setTxLink(txLog.getLastId());
683    else if (message != null && !message.equals("No Message returned")) 
684      return new ValidationResult(IssueSeverity.WARNING, message, new ConceptDefinitionComponent().setDisplay(display)).setTxLink(txLog.getLastId()).setTxLink(txLog.getLastId());
685    else if (display != null)
686      return new ValidationResult(new ConceptDefinitionComponent().setDisplay(display)).setTxLink(txLog.getLastId()).setTxLink(txLog.getLastId());
687    else
688      return new ValidationResult(new ConceptDefinitionComponent()).setTxLink(txLog.getLastId()).setTxLink(txLog.getLastId());
689  }
690
691  // --------------------------------------------------------------------------------------------------------------------------------------------------------
692  
693  public void initTS(String cachePath) throws Exception {
694    txCache = new TerminologyCache(lock, cachePath);
695  }
696
697  @Override
698  public List<ConceptMap> findMapsForSource(String url) throws FHIRException {
699    synchronized (lock) {
700      List<ConceptMap> res = new ArrayList<ConceptMap>();
701      for (ConceptMap map : maps.getList())
702        if (((Reference) map.getSource()).getReference().equals(url)) 
703          res.add(map);
704      return res;
705    }
706  }
707
708  public boolean isCanRunWithoutTerminology() {
709    return canRunWithoutTerminology;
710  }
711
712  public void setCanRunWithoutTerminology(boolean canRunWithoutTerminology) {
713    this.canRunWithoutTerminology = canRunWithoutTerminology;
714  }
715
716  public void setLogger(ILoggingService logger) {
717    this.logger = logger;
718  }
719
720  public Parameters getExpansionParameters() {
721    return expParameters;
722  }
723
724  public void setExpansionProfile(Parameters expParameters) {
725    this.expParameters = expParameters;
726  }
727
728  @Override
729  public boolean isNoTerminologyServer() {
730    return noTerminologyServer;
731  }
732
733  public String getName() {
734    return name;
735  }
736
737  public void setName(String name) {
738    this.name = name;
739  }
740
741  @Override
742  public Set<String> getResourceNamesAsSet() {
743    Set<String> res = new HashSet<String>();
744    res.addAll(getResourceNames());
745    return res;
746  }
747
748  public boolean isAllowLoadingDuplicates() {
749    return allowLoadingDuplicates;
750  }
751
752  public void setAllowLoadingDuplicates(boolean allowLoadingDuplicates) {
753    this.allowLoadingDuplicates = allowLoadingDuplicates;
754  }
755
756  @SuppressWarnings("unchecked")
757  @Override
758  public <T extends Resource> T fetchResourceWithException(Class<T> class_, String uri) throws FHIRException {
759       if (class_ == StructureDefinition.class)
760      uri = ProfileUtilities.sdNs(uri, getOverrideVersionNs());
761    synchronized (lock) {
762
763      String version = null;
764      if (uri.contains("|")) {
765        version = uri.substring(uri.lastIndexOf("|")+1);
766        uri = uri.substring(0, uri.lastIndexOf("|"));
767      }
768      if (uri.contains("#"))
769        uri = uri.substring(0, uri.indexOf("#"));
770      if (class_ == Resource.class || class_ == null) {
771        if (structures.has(uri))
772          return (T) structures.get(uri);
773        if (guides.has(uri))
774          return (T) guides.get(uri);
775        if (capstmts.has(uri))
776          return (T) capstmts.get(uri);
777        if (valueSets.has(uri))
778          return (T) valueSets.get(uri);
779        if (codeSystems.has(uri))
780          return (T) codeSystems.get(uri);
781        if (operations.has(uri))
782          return (T) operations.get(uri);
783        if (searchParameters.has(uri))
784          return (T) searchParameters.get(uri);
785        if (plans.has(uri))
786          return (T) plans.get(uri);
787        if (maps.has(uri))
788          return (T) maps.get(uri);
789        if (transforms.has(uri))
790          return (T) transforms.get(uri);
791        if (questionnaires.has(uri))
792          return (T) questionnaires.get(uri);
793        for (Map<String, Resource> rt : allResourcesById.values()) {
794          for (Resource r : rt.values()) {
795            if (r instanceof CanonicalResource) {
796              CanonicalResource mr = (CanonicalResource) r;
797              if (uri.equals(mr.getUrl()))
798                return (T) mr;
799            }
800          }            
801        }
802        return null;      
803      } else if (class_ == ImplementationGuide.class) {
804        return (T) guides.get(uri);
805      } else if (class_ == CapabilityStatement.class) {
806        return (T) capstmts.get(uri);
807      } else if (class_ == StructureDefinition.class) {
808        return (T) structures.get(uri);
809      } else if (class_ == StructureMap.class) {
810        return (T) transforms.get(uri);
811      } else if (class_ == ValueSet.class) {
812        if (valueSets.has(uri, version))
813          return (T) valueSets.get(uri, version);
814        else
815          return (T) valueSets.get(uri);
816      } else if (class_ == CodeSystem.class) {
817        if (codeSystems.has(uri, version))
818          return (T) codeSystems.get(uri, version);
819        else
820          return (T) codeSystems.get(uri);
821      } else if (class_ == ConceptMap.class) {
822        return (T) maps.get(uri);
823      } else if (class_ == PlanDefinition.class) {
824        return (T) plans.get(uri);
825      } else if (class_ == OperationDefinition.class) {
826        OperationDefinition od = operations.get(uri);
827        return (T) od;
828      } else if (class_ == Questionnaire.class) {
829        return (T) questionnaires.get(uri);
830      } else if (class_ == SearchParameter.class) {
831        SearchParameter res = searchParameters.get(uri);
832        return (T) res;
833      }
834      if (class_ == CodeSystem.class && codeSystems.has(uri))
835        return (T) codeSystems.get(uri);
836      if (class_ == ValueSet.class && valueSets.has(uri))
837        return (T) valueSets.get(uri);
838      
839      if (class_ == Questionnaire.class)
840        return (T) questionnaires.get(uri);
841      if (class_ == null) {
842        if (uri.matches(Constants.URI_REGEX) && !uri.contains("ValueSet"))
843          return null;
844
845        // it might be a special URL.
846        if (Utilities.isAbsoluteUrl(uri) || uri.startsWith("ValueSet/")) {
847          Resource res = null; // findTxValueSet(uri);
848          if (res != null)
849            return (T) res;
850        }
851        return null;      
852      }    
853      if (supportedCodeSystems.contains(uri))
854        return null;
855      throw new FHIRException("not done yet: can't fetch "+uri);
856    }
857  }
858
859  private Set<String> notCanonical = new HashSet<String>();
860
861  private String overrideVersionNs;
862  
863//  private MetadataResource findTxValueSet(String uri) {
864//    MetadataResource res = expansionCache.getStoredResource(uri);
865//    if (res != null)
866//      return res;
867//    synchronized (lock) {
868//      if (notCanonical.contains(uri))
869//        return null;
870//    }
871//    try {
872//      tlog("get canonical "+uri);
873//      res = txServer.getCanonical(ValueSet.class, uri);
874//    } catch (Exception e) {
875//      synchronized (lock) {
876//        notCanonical.add(uri);
877//      }
878//      return null;
879//    }
880//    if (res != null)
881//      try {
882//        expansionCache.storeResource(res);
883//      } catch (IOException e) {
884//      }
885//    return res;
886//  }
887
888  @Override
889  public Resource fetchResourceById(String type, String uri) {
890    synchronized (lock) {
891      String[] parts = uri.split("\\/");
892      if (!Utilities.noString(type) && parts.length == 1) {
893        if (allResourcesById.containsKey(type))
894          return allResourcesById.get(type).get(parts[0]);
895        else
896          return null;
897      }
898      if (parts.length >= 2) {
899        if (!Utilities.noString(type))
900          if (!type.equals(parts[parts.length-2])) 
901            throw new Error("Resource type mismatch for "+type+" / "+uri);
902        return allResourcesById.get(parts[parts.length-2]).get(parts[parts.length-1]);
903      } else
904        throw new Error("Unable to process request for resource for "+type+" / "+uri);
905    }
906  }
907
908  public <T extends Resource> T fetchResource(Class<T> class_, String uri) {
909    try {
910      return fetchResourceWithException(class_, uri);
911    } catch (FHIRException e) {
912      throw new Error(e);
913    }
914  }
915  
916  @Override
917  public <T extends Resource> boolean hasResource(Class<T> class_, String uri) {
918    try {
919      return fetchResourceWithException(class_, uri) != null;
920    } catch (Exception e) {
921      return false;
922    }
923  }
924
925
926  public TranslationServices translator() {
927    return translator;
928  }
929
930  public void setTranslator(TranslationServices translator) {
931    this.translator = translator;
932  }
933  
934  public class NullTranslator implements TranslationServices {
935
936    @Override
937    public String translate(String context, String value, String targetLang) {
938      return value;
939    }
940
941    @Override
942    public String translate(String context, String value) {
943      return value;
944    }
945
946    @Override
947    public String toStr(float value) {
948      return null;
949    }
950
951    @Override
952    public String toStr(Date value) {
953      return null;
954    }
955
956    @Override
957    public String translateAndFormat(String contest, String lang, String value, Object... args) {
958      return String.format(value, args);
959    }
960
961    @Override
962    public Map<String, String> translations(String value) {
963      // TODO Auto-generated method stub
964      return null;
965    }
966
967    @Override
968    public Set<String> listTranslations(String category) {
969      // TODO Auto-generated method stub
970      return null;
971    }
972
973  }
974  
975  public void reportStatus(JsonObject json) {
976    synchronized (lock) {
977      json.addProperty("codeystem-count", codeSystems.size());
978      json.addProperty("valueset-count", valueSets.size());
979      json.addProperty("conceptmap-count", maps.size());
980      json.addProperty("transforms-count", transforms.size());
981      json.addProperty("structures-count", structures.size());
982      json.addProperty("guides-count", guides.size());
983      json.addProperty("statements-count", capstmts.size());
984    }
985  }
986
987
988  public void dropResource(Resource r) throws FHIRException {
989    dropResource(r.fhirType(), r.getId());   
990  }
991
992  public void dropResource(String fhirType, String id) {
993    synchronized (lock) {
994
995      Map<String, Resource> map = allResourcesById.get(fhirType);
996      if (map == null) {
997        map = new HashMap<String, Resource>();
998        allResourcesById.put(fhirType, map);
999      }
1000      if (map.containsKey(id))
1001        map.remove(id);
1002
1003      if (fhirType.equals("StructureDefinition"))
1004        structures.drop(id);
1005      else if (fhirType.equals("ImplementationGuide"))
1006        guides.drop(id);
1007      else if (fhirType.equals("CapabilityStatement"))
1008        capstmts.drop(id);
1009      else if (fhirType.equals("ValueSet"))
1010        valueSets.drop(id);
1011      else if (fhirType.equals("CodeSystem"))
1012        codeSystems.drop(id);
1013      else if (fhirType.equals("OperationDefinition"))
1014        operations.drop(id);
1015      else if (fhirType.equals("Questionnaire"))
1016        questionnaires.drop(id);
1017      else if (fhirType.equals("ConceptMap"))
1018        maps.drop(id);
1019      else if (fhirType.equals("StructureMap"))
1020        transforms.drop(id);
1021      else if (fhirType.equals("NamingSystem"))
1022        for (int i = systems.size()-1; i >= 0; i--) {
1023          if (systems.get(i).getId().equals(id))
1024            systems.remove(i);
1025        }
1026    }
1027  }
1028
1029  private <T extends CanonicalResource> void dropMetadataResource(Map<String, T> map, String id) {
1030    T res = map.get(id);
1031    if (res != null) {
1032      map.remove(id);
1033      if (map.containsKey(res.getUrl()))
1034        map.remove(res.getUrl());
1035      if (res.getVersion() != null)
1036        if (map.containsKey(res.getUrl()+"|"+res.getVersion()))
1037          map.remove(res.getUrl()+"|"+res.getVersion());
1038    }
1039  }
1040
1041  @Override
1042  public List<CanonicalResource> allConformanceResources() {
1043    synchronized (lock) {
1044      List<CanonicalResource> result = new ArrayList<CanonicalResource>();
1045      structures.listAllM(result);
1046      guides.listAllM(result);
1047      capstmts.listAllM(result);
1048      codeSystems.listAllM(result);
1049      valueSets.listAllM(result);
1050      maps.listAllM(result);
1051      transforms.listAllM(result);
1052      plans.listAllM(result);
1053      questionnaires.listAllM(result);
1054      return result;
1055    }
1056  }
1057  
1058  public String listSupportedSystems() {
1059    synchronized (lock) {
1060      String sl = null;
1061      for (String s : supportedCodeSystems)
1062        sl = sl == null ? s : sl + "\r\n" + s;
1063      return sl;
1064    }
1065  }
1066
1067
1068  public int totalCount() {
1069    synchronized (lock) {
1070      return valueSets.size() +  maps.size() + structures.size() + transforms.size();
1071    }
1072  }
1073  
1074  public List<ConceptMap> listMaps() {
1075    List<ConceptMap> m = new ArrayList<ConceptMap>();
1076    synchronized (lock) {
1077      maps.listAll(m);
1078    }
1079    return m;
1080  }
1081  
1082  public List<StructureMap> listTransforms() {
1083    List<StructureMap> m = new ArrayList<StructureMap>();
1084    synchronized (lock) {
1085      transforms.listAll(m);    
1086    }
1087    return m;
1088  }
1089  
1090  public StructureMap getTransform(String code) {
1091    synchronized (lock) {
1092      return transforms.get(code);
1093    }
1094  }
1095
1096  public List<StructureDefinition> listStructures() {
1097    List<StructureDefinition> m = new ArrayList<StructureDefinition>();
1098    synchronized (lock) {
1099      structures.listAll(m);    
1100    }
1101    return m;
1102  }
1103
1104  public StructureDefinition getStructure(String code) {
1105    synchronized (lock) {
1106      return structures.get(code);
1107    }
1108  }
1109
1110  @Override
1111  public String oid2Uri(String oid) {
1112    synchronized (lock) {
1113      if (oid != null && oid.startsWith("urn:oid:")) {
1114        oid = oid.substring(8);
1115      }
1116
1117      String uri = OIDUtils.getUriForOid(oid);
1118      if (uri != null)
1119        return uri;
1120      for (NamingSystem ns : systems) {
1121        if (hasOid(ns, oid)) {
1122          uri = getUri(ns);
1123          if (uri != null)
1124            return null;
1125        }
1126      }
1127    }
1128    return null;
1129  }
1130  
1131
1132  private String getUri(NamingSystem ns) {
1133    for (NamingSystemUniqueIdComponent id : ns.getUniqueId()) {
1134      if (id.getType() == NamingSystemIdentifierType.URI)
1135        return id.getValue();
1136    }
1137    return null;
1138  }
1139
1140  private boolean hasOid(NamingSystem ns, String oid) {
1141    for (NamingSystemUniqueIdComponent id : ns.getUniqueId()) {
1142      if (id.getType() == NamingSystemIdentifierType.OID && id.getValue().equals(oid))
1143        return true;
1144    }
1145    return false;
1146  }
1147
1148  public void cacheVS(JsonObject json, Map<String, ValidationResult> t) {
1149    synchronized (lock) {
1150      validationCache.put(json.get("url").getAsString(), t);
1151    }
1152  }
1153
1154  public SearchParameter getSearchParameter(String code) {
1155    synchronized (lock) {
1156      return searchParameters.get(code);
1157    }
1158  }
1159
1160  @Override
1161  public String getOverrideVersionNs() {
1162    return overrideVersionNs;
1163  }
1164
1165  @Override
1166  public void setOverrideVersionNs(String value) {
1167    overrideVersionNs = value;
1168  }
1169
1170  @Override
1171  public ILoggingService getLogger() {
1172    return logger;
1173  }
1174
1175  @Override
1176  public StructureDefinition fetchTypeDefinition(String typeName) {
1177    if (Utilities.isAbsoluteUrl(typeName))
1178      return fetchResource(StructureDefinition.class, typeName);
1179    else
1180      return fetchResource(StructureDefinition.class, "http://hl7.org/fhir/StructureDefinition/"+typeName);
1181  }
1182
1183  public boolean isTlogging() {
1184    return tlogging;
1185  }
1186
1187  public void setTlogging(boolean tlogging) {
1188    this.tlogging = tlogging;
1189  }
1190
1191  public UcumService getUcumService() {
1192    return ucumService;
1193  }
1194
1195  public void setUcumService(UcumService ucumService) {
1196    this.ucumService = ucumService;
1197  }
1198
1199  @Override
1200  public List<StructureDefinition> getStructures() {
1201    List<StructureDefinition> res = new ArrayList<>();
1202    synchronized (lock) { // tricky, because you need to lock this as well, but it's really not in use yet
1203      structures.listAll(res);
1204    }
1205    return res;
1206  }
1207  
1208  public String getLinkForUrl(String corePath, String url) {
1209    if (codeSystems.has(url))
1210      return codeSystems.get(url).getUserString("path");
1211
1212    if (valueSets.has(url))
1213      return valueSets.get(url).getUserString("path");
1214
1215    if (maps.has(url))
1216      return maps.get(url).getUserString("path");
1217    
1218    if (transforms.has(url))
1219      return transforms.get(url).getUserString("path");
1220    
1221    if (structures.has(url))
1222      return structures.get(url).getUserString("path");
1223    
1224    if (guides.has(url))
1225      return guides.get(url).getUserString("path");
1226    
1227    if (capstmts.has(url))
1228      return capstmts.get(url).getUserString("path");
1229
1230    if (searchParameters.has(url))
1231      return searchParameters.get(url).getUserString("path");
1232        
1233    if (questionnaires.has(url))
1234      return questionnaires.get(url).getUserString("path");
1235
1236    if (operations.has(url))
1237      return operations.get(url).getUserString("path");
1238    
1239    if (plans.has(url))
1240      return plans.get(url).getUserString("path");
1241
1242    if (url.equals("http://loinc.org"))
1243      return corePath+"loinc.html";
1244    if (url.equals("http://unitsofmeasure.org"))
1245      return corePath+"ucum.html";
1246    if (url.equals("http://snomed.info/sct"))
1247      return corePath+"snomed.html";
1248    return null;
1249  }
1250
1251  
1252
1253  public List<ImplementationGuide> allImplementationGuides() {
1254    List<ImplementationGuide> res = new ArrayList<>();
1255    guides.listAll(res);
1256    return res;
1257  }
1258
1259  @Override
1260  public Map<String, byte[]> getBinaries() {
1261    return binaries;
1262  }
1263
1264
1265}