001package org.hl7.fhir.utilities.cache;
002
003/*-
004 * #%L
005 * org.hl7.fhir.utilities
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.BufferedOutputStream;
025import java.io.ByteArrayOutputStream;
026import java.io.File;
027import java.io.FileInputStream;
028import java.io.IOException;
029import java.io.InputStream;
030import java.net.HttpURLConnection;
031import java.net.URL;
032import java.net.URLConnection;
033import java.sql.Timestamp;
034import java.time.Instant;
035import java.util.ArrayList;
036import java.util.Collections;
037import java.util.HashMap;
038import java.util.List;
039import java.util.Map;
040import java.util.Map.Entry;
041
042import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
043import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
044import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
045import org.apache.commons.io.FileUtils;
046import org.hl7.fhir.exceptions.FHIRException;
047import org.hl7.fhir.utilities.IniFile;
048import org.hl7.fhir.utilities.TextFile;
049import org.hl7.fhir.utilities.Utilities;
050import org.hl7.fhir.utilities.json.JSONUtil;
051
052import com.google.gson.GsonBuilder;
053import com.google.gson.JsonArray;
054import com.google.gson.JsonElement;
055import com.google.gson.JsonObject;
056
057/**
058 * Package cache manager
059 * 
060 * API:
061 * 
062 * constructor
063 *  getPackageUrl
064 *  getPackageId
065 *  findPackageCache 
066 *  addPackageToCache  
067 * 
068 * @author Grahame Grieve
069 *
070 */
071public class PackageCacheManager {
072
073  public class VersionHistory {
074    private String id;
075    private String canonical;
076    private String current;
077    private Map<String, String> versions = new HashMap<>();
078    public String getCanonical() {
079      return canonical;
080    }
081    public String getCurrent() {
082      return current;
083    }
084    public Map<String, String> getVersions() {
085      return versions;
086    }
087    public String getId() {
088      return id;
089    } 
090  }
091
092
093  public class PackageEntry {
094
095    private byte[] bytes;
096    private String name;
097
098    public PackageEntry(String name) {
099      this.name = name;
100    }
101
102    public PackageEntry(String name, byte[] bytes) {
103      this.name = name;
104      this.bytes = bytes;
105    }
106  }
107
108  public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$";
109  public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$";
110
111  private static final int BUFFER_SIZE = 1024;
112  private static final String CACHE_VERSION = "3"; // second version - see wiki page
113  private static final int ANALYSIS_VERSION = 2;
114
115  private String cacheFolder;
116  private boolean buildLoaded;
117  private JsonArray buildInfo;
118  private boolean progress = true;
119  private List<NpmPackage> temporaryPackages = new ArrayList<NpmPackage>();
120  private Map<String, String> ciList = new HashMap<String, String>();
121  private List<String> allUrls;
122  private Map<String, VersionHistory> historyCache = new HashMap<>();
123  
124  public PackageCacheManager(boolean userMode, int toolsVersion) throws IOException {
125    if (userMode)
126      cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
127    else
128      cacheFolder = Utilities.path("var", "lib", ".fhir", "packages");
129    if (!(new File(cacheFolder).exists()))
130      Utilities.createDirectory(cacheFolder);
131    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
132      TextFile.stringToFile("[cache]\r\nversion="+CACHE_VERSION+"\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false);  
133    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
134    boolean save = false;
135    String v = ini.getStringProperty("cache", "version");    
136    if (!CACHE_VERSION.equals(v)) {
137      clearCache();
138      ini.setStringProperty("cache", "version", CACHE_VERSION, null);
139      save = true;
140    }
141    save = initUrlMaps(ini, save);
142    if (save) {
143      if (!CACHE_VERSION.equals(ini.getStringProperty("cache", "version"))) {
144        throw new Error("what?");
145      }
146      ini.save();
147    }
148  }
149
150  public static String userDir() throws IOException {
151    return Utilities.path(System.getProperty("user.home"), ".fhir", "packages");
152  }
153
154  // ========================= Utilities ============================================================================
155  
156  private List<String> sorted(String[] keys) {
157    List<String> names = new ArrayList<String>();
158    for (String s : keys)
159      names.add(s);
160    Collections.sort(names);
161    return names;
162  }
163
164  private NpmPackage loadPackageInfo(String path) throws IOException {
165    NpmPackage pi = new NpmPackage(path);
166    return pi;
167  }
168
169  public String getFolder() {
170    return cacheFolder;
171  }
172
173  private void analysePackage(String dir) throws IOException {
174    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
175    indexer.start();
176    int i = 0;
177    int c = 0;
178    File[] packages = new File(Utilities.path(dir, "package")).listFiles();
179    for (File f : packages) {
180      indexer.seeFile(f.getName(), TextFile.fileToBytes(f));
181      i++;
182      if (progress && i % 50 == 0) {
183        c++;
184        System.out.print(".");
185        if (c == 120) {
186          System.out.println("");
187          System.out.print("  ");
188          c = 2;
189        }
190      }    
191    }
192    TextFile.stringToFile(indexer.build(), Utilities.path(dir, "package", ".index.json"));
193  }
194
195
196
197  private NpmPackage checkCurrency(String id, NpmPackage p) {
198    // special case: current versions roll over, and we have to check their currency
199    try {
200      String url = ciList.get(id);
201      JsonObject json = fetchJson(Utilities.pathURL(url, "package.manifest.json"));
202      String currDate = JSONUtil.str(json, "date");
203      String packDate = p.date();
204      if (!currDate.equals(packDate))
205        return null; // nup, we need a new copy 
206      return p;
207    } catch (Exception e) {
208      return p;
209    }
210  }
211  
212  private JsonObject fetchJson(String source) throws IOException {
213    URL url = new URL(source);
214    URLConnection c = url.openConnection();
215    return (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(c.getInputStream()));
216  }
217  
218  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
219    try {
220      URL url = new URL(source);
221      URLConnection c = url.openConnection();
222      return c.getInputStream();
223    } catch (Exception e) {
224      if (optional)
225        return null;
226      else
227        throw new FHIRException(e.getMessage(), e);
228    }
229  }
230
231  // ========================= Full Cache Management ============================================================================
232
233  private void clearCache() throws IOException {
234    for (File f : new File(cacheFolder).listFiles()) {
235      if (f.isDirectory())
236        FileUtils.deleteDirectory(f);
237      else if (!f.getName().equals("packages.ini"))
238        FileUtils.forceDelete(f);
239    }    
240    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
241    ini.removeSection("packages");
242    ini.save();
243  }
244
245//  private void checkDeleteVersion(String id, String ver, int minVer) {
246//    if (hasPackage(id, ver)) {
247//      boolean del = true;
248//      NpmPackage pck;
249//      try {
250//        pck = loadPackageFromCacheOnly(id, ver);
251//        if (pck.getNpm().has("tools-version")) {
252//          del = pck.getNpm().get("tools-version").getAsInt() < minVer;
253//        }
254//      } catch (Exception e) {
255//      }
256//      if (del)
257//        try {
258//          removePackage(id, ver);
259//        } catch (IOException e) {
260//        }
261//    }
262//  }
263  
264//private void convertPackageCacheFrom1To2() throws IOException {
265//for (File f : new File(cacheFolder).listFiles()) {
266//  if (f.isDirectory() && f.getName().contains("-")) {
267//    String s = f.getName();
268//    int i = s.lastIndexOf("-");
269//    s = s.substring(0, i)+"#"+s.substring(i+1);
270//    File nf = new File(Utilities.path(cacheFolder, s));
271//    if (!f.renameTo(nf))
272//      throw new IOException("Unable to rename "+f.getAbsolutePath()+" to "+nf.getAbsolutePath());
273//  }
274//}
275//}
276  
277  // ========================= URL maps (while waiting for package registry) ============================================================================
278  
279  public boolean initUrlMaps(IniFile ini, boolean save) {
280    save = checkIniHasMapping("hl7.fhir.core", "http://hl7.org/fhir", ini) || save;
281    
282    save = checkIniHasMapping("hl7.fhir.r2.core", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.core.tgz", ini) || save;
283    save = checkIniHasMapping("hl7.fhir.r2.examples", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.examples.tgz", ini) || save;
284    save = checkIniHasMapping("hl7.fhir.r2.elements", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.elements.tgz", ini) || save;
285    save = checkIniHasMapping("hl7.fhir.r2.expansions", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.expansions.tgz", ini) || save;
286    save = checkIniHasMapping("hl7.fhir.r2b.core", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.core.tgz", ini) || save;
287    save = checkIniHasMapping("hl7.fhir.r2b.examples", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.examples.tgz", ini) || save;
288    save = checkIniHasMapping("hl7.fhir.r2b.elements", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.elements.tgz", ini) || save;
289    save = checkIniHasMapping("hl7.fhir.r2b.expansions", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.expansions.tgz", ini) || save;
290    save = checkIniHasMapping("hl7.fhir.r3.core", "http://hl7.org/fhir/STU3/hl7.fhir.r3.core.tgz", ini) || save;
291    save = checkIniHasMapping("hl7.fhir.r3.examples", "http://hl7.org/fhir/STU3/hl7.fhir.r3.examples.tgz", ini) || save;
292    save = checkIniHasMapping("hl7.fhir.r3.elements", "http://hl7.org/fhir/STU3/hl7.fhir.r3.elements.tgz", ini) || save;
293    save = checkIniHasMapping("hl7.fhir.r3.expansions", "http://hl7.org/fhir/STU3/hl7.fhir.r3.expansions.tgz", ini) || save;
294    save = checkIniHasMapping("hl7.fhir.r4.core", "http://hl7.org/fhir/R4/hl7.fhir.r4.core.tgz", ini) || save;
295    save = checkIniHasMapping("hl7.fhir.r4.examples", "http://hl7.org/fhir/R4/hl7.fhir.r4.examples.tgz", ini) || save;
296    save = checkIniHasMapping("hl7.fhir.r4.elements", "http://hl7.org/fhir/R4/hl7.fhir.r4.elements.tgz", ini) || save;
297    save = checkIniHasMapping("hl7.fhir.r4.expansions", "http://hl7.org/fhir/R4/hl7.fhir.r4.expansions.tgz", ini) || save;
298
299    save = checkIniHasMapping("hl7.fhir.r5.core", "http://build.fhir.org/package.tgz", ini) || save;
300
301    save = checkIniHasMapping("fhir.argonaut.ehr", "http://fhir.org/guides/argonaut", ini) || save;
302    save = checkIniHasMapping("fhir.argonaut.pd", "http://fhir.org/guides/argonaut-pd", ini) || save;
303    save = checkIniHasMapping("fhir.argonaut.scheduling", "http://fhir.org/guides/argonaut-scheduling", ini) || save;
304    save = checkIniHasMapping("fhir.hspc.acog", "http://hl7.org/fhir/fpar", ini) || save;
305    save = checkIniHasMapping("hl7.fhir.au.argonaut", "http://hl7.org.au/fhir/argonaut", ini) || save;
306    save = checkIniHasMapping("hl7.fhir.au.base", "http://hl7.org.au/fhir/base", ini) || save;
307    save = checkIniHasMapping("hl7.fhir.au.pd", "http://hl7.org.au/fhir/pd", ini) || save;
308    save = checkIniHasMapping("hl7.fhir.smart", "http://hl7.org/fhir/smart-app-launch", ini) || save;
309    save = checkIniHasMapping("hl7.fhir.snomed", "http://hl7.org/fhir/ig/snomed", ini) || save;
310    save = checkIniHasMapping("hl7.fhir.test.v10", "http://hl7.org/fhir/test-ig-10", ini) || save;
311    save = checkIniHasMapping("hl7.fhir.test.v30", "http://hl7.org/fhir/test", ini) || save;
312    save = checkIniHasMapping("hl7.fhir.us.breastcancer", "http://hl7.org/fhir/us/breastcancer", ini) || save;
313    save = checkIniHasMapping("hl7.fhir.us.ccda", "http://hl7.org/fhir/us/ccda", ini) || save;
314    save = checkIniHasMapping("hl7.fhir.us.cds.opioids", "http://hl7.org/fhir/ig/opioid-cds", ini) || save;
315    save = checkIniHasMapping("hl7.fhir.us.core", "http://hl7.org/fhir/us/core", ini) || save;
316    save = checkIniHasMapping("hl7.fhir.us.dafresearch", "http://hl7.org/fhir/us/daf-research", ini) || save;
317    save = checkIniHasMapping("hl7.fhir.us.ecr", "http://fhir.hl7.org/us/ecr", ini) || save;
318    save = checkIniHasMapping("hl7.fhir.us.hai", "http://hl7.org/fhir/us/hai", ini) || save;
319    save = checkIniHasMapping("hl7.fhir.us.hedis", "http://hl7.org/fhir/us/hedis", ini) || save;
320    save = checkIniHasMapping("hl7.fhir.us.meds", "http://hl7.org/fhir/us/meds", ini) || save;
321    save = checkIniHasMapping("hl7.fhir.us.qicore", "http://hl7.org/fhir/us/qicore", ini) || save;
322    save = checkIniHasMapping("hl7.fhir.us.sdc", "http://hl7.org/fhir/us/sdc", ini) || save;
323    save = checkIniHasMapping("hl7.fhir.us.sdcde", "http://hl7.org/fhir/us/sdcde", ini) || save;
324    save = checkIniHasMapping("hl7.fhir.uv.genomicsreporting", "http://hl7.org/fhir/uv/genomics-reporting", ini) || save;
325    save = checkIniHasMapping("hl7.fhir.uv.immds", "http://hl7.org/fhir/uv/cdsi", ini) || save;
326    save = checkIniHasMapping("hl7.fhir.uv.ips", "http://hl7.org/fhir/uv/ips", ini) || save;
327    save = checkIniHasMapping("hl7.fhir.uv.phd", "http://hl7.org/fhir/devices", ini) || save;
328    save = checkIniHasMapping("hl7.fhir.uv.vhdir", "http://hl7.org/fhir/ig/vhdir", ini) || save;
329    save = checkIniHasMapping("hl7.fhir.vn.base", "http://hl7.org/fhir/ig/vietnam", ini) || save;
330    save = checkIniHasMapping("hl7.fhir.vocabpoc", "http://hl7.org/fhir/ig/vocab-poc", ini) || save;
331    return save;
332  }
333
334  private boolean checkIniHasMapping(String pid, String curl, IniFile ini) {
335    if (curl.equals(ini.getStringProperty("urls", pid)))
336      return false;
337    ini.setStringProperty("urls", pid, curl, null);
338    return true;
339  }
340
341  public void recordMap(String url, String id) throws IOException {
342    if (url == null)
343      return;
344    
345    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
346        throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #1");
347    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
348    ini.setStringProperty("urls", id, url, null);
349    if (!CACHE_VERSION.equals(ini.getStringProperty("cache", "version"))) {
350      throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" cache version mismatch: expected '"+CACHE_VERSION+"', found '"+ini.getStringProperty("cache", "version")+"'");
351    }
352    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
353      throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #2");
354
355    ini.save();
356    if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists()))
357      throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #3");
358  }
359
360  public String getPackageUrl(String id) throws IOException {
361    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
362    return ini.getStringProperty("urls", id); 
363  }
364  
365  public String getPackageId(String url) throws IOException {
366    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
367    String[] ids = ini.getPropertyNames("urls");
368    if (ids != null) {
369      for (String id : ids) {
370        if (url.equals(ini.getStringProperty("urls", id)))
371          return id;
372      }
373    }
374    return null;
375  }
376  
377  public JsonArray loadFromBuildServer() throws IOException {
378    buildLoaded = true; // whether it succeeds or not
379    URL url = new URL("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
380    HttpURLConnection connection = (HttpURLConnection) url.openConnection();
381    connection.setRequestMethod("GET");
382    InputStream json = connection.getInputStream();
383    buildInfo = (JsonArray) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
384  
385    for (JsonElement n : buildInfo) {
386      JsonObject o = (JsonObject) n;
387      if (o.has("url") && o.has("package-id") && o.get("package-id").getAsString().contains(".")) {
388        String u = o.get("url").getAsString();
389        if (u.contains("/ImplementationGuide/"))
390          u = u.substring(0, u.indexOf("/ImplementationGuide/"));
391        recordMap(u, o.get("package-id").getAsString());
392        ciList.put(o.get("package-id").getAsString(), "https://build.fhir.org/ig/"+o.get("repo").getAsString());
393      }
394    }
395    return buildInfo;
396  }
397
398  public boolean isBuildLoaded() {
399    return buildLoaded;
400  }
401
402
403  public String buildPath(String url) {
404    for (JsonElement e : buildInfo) {
405      JsonObject j = (JsonObject) e;
406      if (j.has("url") && (url.equals(j.get("url").getAsString()) || j.get("url").getAsString().startsWith(url+"/ImplementationGuide"))) {
407        return "https://build.fhir.org/ig/"+j.get("repo").getAsString();
408      }
409    }
410    return null;
411  }
412 
413  public boolean checkBuildLoaded() throws IOException {
414    if (isBuildLoaded())
415      return true;
416    loadFromBuildServer();
417    return false;
418  }
419  
420  public Map<String, String> getCiList() {
421    return ciList;
422  }
423
424  public List<String> getUrls() throws IOException {
425    if (allUrls == null)
426    {
427      IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
428      allUrls = new ArrayList<>();
429      for (String s : ini.getPropertyNames("urls"))
430        allUrls.add(ini.getStringProperty("urls", s));
431      try {
432        URL url = new URL("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json?nocache=" + System.currentTimeMillis());
433        HttpURLConnection connection = (HttpURLConnection) url.openConnection();
434        connection.setRequestMethod("GET");
435        InputStream json = connection.getInputStream();
436        JsonObject packages = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
437        JsonArray guides = packages.getAsJsonArray("guides");
438        for (JsonElement g : guides) {
439          JsonObject gi = (JsonObject) g;
440          if (gi.has("canonical"))
441            if (!allUrls.contains(gi.get("canonical").getAsString()))
442              allUrls.add(gi.get("canonical").getAsString());
443        }
444      } catch (Exception e) {
445        System.out.println("Listing known Implementation Guides failed: "+e.getMessage());
446      }
447    }
448    return allUrls;    
449  }
450
451  // ========================= Package API ============================================================================
452
453  public void removePackage(String id, String ver) throws IOException  {
454    String f = Utilities.path(cacheFolder, id+"#"+ver);
455    Utilities.clearDirectory(f);    
456    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
457    ini.removeProperty("packages", id+"#"+ver);
458    ini.save();
459    new File(f).delete();
460  }
461
462  /**
463   * get the latest version of the package from what is in the cache
464   * @param id
465   * @return
466   * @throws IOException
467   */
468  public NpmPackage loadPackageFromCacheOnly(String id) throws IOException {
469    List<String> l = sorted(new File(cacheFolder).list());
470    for (int i = l.size()-1; i >= 0; i--) {
471      String f = l.get(i);
472      if (f.startsWith(id+"#")) {
473        return loadPackageInfo(Utilities.path(cacheFolder, f)); 
474      }
475    }
476    return null;    
477  }
478  
479  /** 
480   * Load the identified package from the cache - it it exists 
481   * 
482   * This is for special purpose only. Generally, use the loadPackage method
483   *  
484   * @param id
485   * @param version
486   * @return
487   * @throws IOException
488   */
489  public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException {
490    for (NpmPackage p : temporaryPackages) {
491      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version)))
492        return p;
493    }
494    for (String f : sorted(new File(cacheFolder).list())) {
495      if (f.equals(id+"#"+version)) {
496        return loadPackageInfo(Utilities.path(cacheFolder, f)); 
497      }
498    }
499    if ("dev".equals(version))
500      return loadPackageFromCacheOnly(id, "current");
501    else
502      return null;
503  }
504
505  /**
506   * Add an already fetched package to the cache
507   */
508  public NpmPackage addPackageToCache(String id, String version, InputStream tgz) throws IOException {
509    if (progress ) {
510      System.out.println("Installing "+id+"#"+(version == null ? "?" : version)+" to the package cache");
511      System.out.print("  Fetching:");
512    }
513    
514    NpmPackage npm = NpmPackage.fromPackage(tgz, true);
515    
516    recordMap(npm.canonical(), npm.name());
517    
518    if (progress )
519      System.out.print("|");
520    if (npm.name() == null || id == null || !id.equals(npm.name())) {
521      if (!id.equals("hl7.fhir.r5.core")) {// temporary work around
522        throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name());
523      }
524    }
525    if (version == null)
526      version = npm.version();
527    
528    String packRoot = Utilities.path(cacheFolder, id+"#"+version);
529    Utilities.createDirectory(packRoot);
530    Utilities.clearDirectory(packRoot);
531      
532    int i = 0;
533    int c = 0;
534    int size = 0;
535    for (Entry<String, byte[]> e : npm.getContent().entrySet()) {
536      String fn = Utilities.path(packRoot, e.getKey());
537      String dir = Utilities.getDirectoryForFile(fn);
538      if (!(new File(dir).exists()))
539        Utilities.createDirectory(dir);
540      TextFile.bytesToFile(e.getValue(), fn);
541      size = size + e.getValue().length;
542      i++;
543      if (progress && i % 50 == 0) {
544        c++;
545        System.out.print(".");
546        if (c == 120) {
547          System.out.println("");
548          System.out.print("  ");
549          c = 2;
550        }
551      }    
552    }
553    if (progress) {
554      System.out.println("");
555      System.out.print("  Analysing");
556    }      
557    
558    analysePackage(packRoot); // do this whether there's an index.json or not.
559    
560    IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini"));
561    ini.setTimeStampFormat("yyyyMMddhhmmss");
562    ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null);
563    ini.setIntegerProperty("package-sizes", id+"#"+version, size, null);
564    ini.save();
565    if (progress)
566      System.out.println(" done.");
567
568    //todo: load dependencies
569    NpmPackage pck = loadPackageInfo(packRoot);
570    if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !version.equals(JSONUtil.str(npm.getNpm(), "version"))) {
571      if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) {
572        npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name"));
573        npm.getNpm().remove("name");
574        npm.getNpm().addProperty("name", id);
575      }
576      if (!version.equals(JSONUtil.str(npm.getNpm(), "version"))) {
577        npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version"));
578        npm.getNpm().remove("version");
579        npm.getNpm().addProperty("version", version);
580      }
581      TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false);
582    }
583
584    return pck;
585  }
586
587  public NpmPackage loadPackage(String id) throws FHIRException, IOException {
588    throw new Error("Not done yet");
589  }
590  
591  public NpmPackage loadPackage(String id, String v) throws FHIRException, IOException {
592    NpmPackage p = loadPackageFromCacheOnly(id, v);
593    if (p != null) {
594      if ("current".equals(v)) {
595        p = checkCurrency(id, p);
596      }
597      if (p != null)
598        return p;
599    }
600
601    if ("dev".equals(v)) {
602      p = loadPackageFromCacheOnly(id, "current");
603      p = checkCurrency(id, p);
604      if (p != null)
605        return p;
606      v = "current";
607    }
608
609    String url = getPackageUrl(id);
610    if (url == null)
611      throw new FHIRException("Unable to resolve the package '"+id+"'");
612    if (url.contains(".tgz")) {
613      InputStream stream = fetchFromUrlSpecific(url, true);
614      if (stream != null)
615        return addPackageToCache(id, v, stream);
616      throw new FHIRException("Unable to find the package source for '"+id+"' at "+url);      
617    }
618    if (v == null) {
619      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(url, "package.tgz"), true);
620      if (stream == null && isBuildLoaded()) { 
621        stream = fetchFromUrlSpecific(Utilities.pathURL(buildPath(url), "package.tgz"), true);
622      }
623      if (stream != null)
624        return addPackageToCache(id, null,  stream);
625      throw new FHIRException("Unable to find the package source for '"+id+"' at "+url);
626    } else if ("current".equals(v) && ciList.containsKey(id)){
627      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), true);
628      return addPackageToCache(id, v, stream);
629    } else {
630      String pu = Utilities.pathURL(url, "package-list.json");
631      JsonObject json;
632      try {
633        json = fetchJson(pu);
634      } catch (Exception e) {
635        throw new FHIRException("Error fetching package list for "+id+" from "+pu+": "+e.getMessage(), e);
636      }
637      if (!id.equals(JSONUtil.str(json, "package-id")))
638        throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id"));
639      for (JsonElement e : json.getAsJsonArray("list")) {
640        JsonObject vo = (JsonObject) e;
641        if (v.equals(JSONUtil.str(vo, "version"))) {
642          InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz"), true);
643          if (stream == null)
644            throw new FHIRException("Unable to find the package source for '"+id+"#"+v+"' at "+Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz"));
645          return addPackageToCache(id, v, stream);
646        }
647      }
648//      // special case: current version
649//      if (id.equals("hl7.fhir.core") && v.equals(currentVersion)) {
650//        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", "package.tgz"), true);
651//        if (stream == null)
652//          throw new FHIRException("Unable to find the package source for '"+id+"#"+v+"' at "+Utilities.pathURL("http://build.fhir.org", "package.tgz"));
653//        return addPackageToCache(id, v, stream);
654//      }
655      throw new FHIRException("Unable to resolve version "+v+" for package "+id);
656    }
657  }
658
659
660//  public void loadFromFolder(String packagesFolder) throws IOException {
661//    for (File f : new File(packagesFolder).listFiles()) {
662//      if (f.getName().endsWith(".tgz")) {
663//        temporaryPackages.add(extractLocally(new FileInputStream(f), f.getName()));
664//      }
665//    }
666//  }
667//
668
669  /**
670   * turn true if a package exists
671   * @param id
672   * @param version
673   * @return
674   */
675  public boolean hasPackage(String id, String version) {
676    for (NpmPackage p : temporaryPackages) {
677      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version)))
678        return true;
679    }
680    for (String f : sorted(new File(cacheFolder).list())) {
681      if (f.equals(id+"#"+version)) {
682        return true; 
683      }
684    }
685    if ("dev".equals(version))
686      return hasPackage(id, "current");
687    else
688      return false;
689  }
690
691
692  /**
693   * List which versions of a package are available
694   * 
695   * @param url
696   * @return
697   * @throws IOException
698   */
699
700  public VersionHistory listVersions(String url) throws IOException {
701    if (historyCache.containsKey(url))
702      return historyCache.get(url);
703    
704    URL url1 = new URL(Utilities.pathURL(url, "package-list.json")+"?nocache=" + System.currentTimeMillis());
705    HttpURLConnection connection = (HttpURLConnection) url1.openConnection();
706    connection.setRequestMethod("GET");
707    InputStream json = connection.getInputStream();
708    JsonObject packageList = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json));
709    VersionHistory res = new VersionHistory();
710    res.id = JSONUtil.str(packageList, "package-id");
711    res.canonical = JSONUtil.str(packageList, "canonical");
712    for (JsonElement j : packageList.getAsJsonArray("list")) {
713      JsonObject jo = (JsonObject) j;
714      if ("current".equals(JSONUtil.str(jo, "version")))
715        res.current = JSONUtil.str(jo, "path");
716      else
717        res.versions.put(JSONUtil.str(jo, "version"), JSONUtil.str(jo, "path"));
718    }
719    historyCache.put(url, res);
720    return res;
721  }
722
723
724  /**
725   * Clear the cache
726   * 
727   * @throws IOException
728   */
729  public void clear() throws IOException {
730    clearCache();
731  }
732    
733  public void loadFromFolder(String packagesFolder) throws IOException {
734    for (File f : new File(packagesFolder).listFiles()) {
735      if (f.getName().endsWith(".tgz")) {
736        temporaryPackages.add(NpmPackage.fromPackage(new FileInputStream(f)));
737      }
738    }
739  }
740}