001package org.hl7.fhir.utilities.npm;
002
003/*
004  Copyright (c) 2011+, HL7, Inc.
005  All rights reserved.
006  
007  Redistribution and use in source and binary forms, with or without modification, 
008  are permitted provided that the following conditions are met:
009    
010   * Redistributions of source code must retain the above copyright notice, this 
011     list of conditions and the following disclaimer.
012   * Redistributions in binary form must reproduce the above copyright notice, 
013     this list of conditions and the following disclaimer in the documentation 
014     and/or other materials provided with the distribution.
015   * Neither the name of HL7 nor the names of its contributors may be used to 
016     endorse or promote products derived from this software without specific 
017     prior written permission.
018  
019  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
020  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
021  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
022  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
023  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
024  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
025  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
026  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
027  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
028  POSSIBILITY OF SUCH DAMAGE.
029  
030 */
031
032
033
034import java.io.BufferedOutputStream;
035import java.io.ByteArrayInputStream;
036import java.io.ByteArrayOutputStream;
037import java.io.File;
038import java.io.FileInputStream;
039import java.io.FileNotFoundException;
040import java.io.FileOutputStream;
041import java.io.IOException;
042import java.io.InputStream;
043import java.io.OutputStream;
044import java.nio.charset.StandardCharsets;
045import java.text.ParseException;
046import java.text.SimpleDateFormat;
047import java.util.ArrayList;
048import java.util.Collections;
049import java.util.Comparator;
050import java.util.Date;
051import java.util.HashMap;
052import java.util.List;
053import java.util.Map;
054import java.util.Set;
055import java.util.UUID;
056import java.util.zip.Deflater;
057import java.util.zip.ZipEntry;
058import java.util.zip.ZipInputStream;
059
060import javax.annotation.Nonnull;
061
062import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
063import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
064import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
065import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
066import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
067import org.apache.commons.compress.compressors.gzip.GzipParameters;
068import org.hl7.fhir.exceptions.FHIRException;
069import org.hl7.fhir.utilities.ByteProvider;
070import org.hl7.fhir.utilities.CommaSeparatedStringBuilder;
071import org.hl7.fhir.utilities.TextFile;
072import org.hl7.fhir.utilities.Utilities;
073import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
074import org.hl7.fhir.utilities.http.HTTPResult;
075import org.hl7.fhir.utilities.http.ManagedWebAccess;
076import org.hl7.fhir.utilities.json.JsonException;
077import org.hl7.fhir.utilities.json.model.JsonArray;
078import org.hl7.fhir.utilities.json.model.JsonElement;
079import org.hl7.fhir.utilities.json.model.JsonObject;
080import org.hl7.fhir.utilities.json.model.JsonProperty;
081import org.hl7.fhir.utilities.json.parser.JsonParser;
082import org.hl7.fhir.utilities.npm.PackageGenerator.PackageType;
083
084/**
085 * info and loader for a package 
086 * 
087 * Packages may exist on disk in the cache, or purely in memory when they are loaded on the fly
088 * 
089 * Packages are contained in subfolders (see the package spec). The FHIR resources will be in "package"
090 * 
091 * @author Grahame Grieve
092 *
093 */
094public class NpmPackage {
095
096  public interface ITransformingLoader {
097
098    byte[] load(File f);
099
100  }
101
102  public class PackageResourceInformationSorter implements Comparator<PackageResourceInformation> {
103    @Override
104    public int compare(PackageResourceInformation o1, PackageResourceInformation o2) {
105      return o1.filename.compareTo(o2.filename);
106    }
107  }
108  
109  public class PackageResourceInformation {
110    private String id;
111    private String resourceType;
112    private String url;
113    private String version;
114    private String filename;
115    private String supplements;
116    private String stype;
117    private String derivation;
118    private String content;
119    
120    public PackageResourceInformation(String root, JsonObject fi) throws IOException {
121      super();
122      id = fi.asString("id");
123      resourceType = fi.asString("resourceType");
124      url = fi.asString("url");
125      version = fi.asString("version");
126      filename = Utilities.path(root, fi.asString("filename"));
127      supplements = fi.asString("supplements");
128      stype = fi.asString("type");
129      derivation = fi.asString("derivation");
130      content = fi.asString("content");
131    }
132    public String getId() {
133      return id;
134    }
135    public String getResourceType() {
136      return resourceType;
137    }
138    public String getStatedType() {
139      return stype;
140    }
141    public String getUrl() {
142      return url;
143    }
144    public String getVersion() {
145      return version;
146    }
147    public String getFilename() {
148      return filename;
149    }
150    public String getSupplements() {
151      return supplements;
152    }
153    public boolean hasId() {
154      return !Utilities.noString(id);
155    }
156    public String getDerivation() {
157      return derivation;
158    }
159    public String getContent() {
160      return content;
161    }
162    
163  }
164  public class IndexVersionSorter implements Comparator<JsonObject> {
165
166    @Override
167    public int compare(JsonObject o0, JsonObject o1) {
168      String v0 = o0.asString("version"); 
169      String v1 = o1.asString("version"); 
170      return v0.compareTo(v1);
171    }
172  }
173
174  public static boolean isValidName(String pid) {
175    return pid.matches("^[a-z][a-zA-Z0-9]*(\\.[a-z][a-zA-Z0-9\\-]*)+$");
176  }
177
178  public static boolean isValidVersion(String ver) {
179    return ver.matches("^[0-9]+\\.[0-9]+\\.[0-9]+$");
180  }
181
182  public class NpmPackageFolder {
183    private String folderName;
184    private Map<String, List<String>> types;
185    private Map<String, byte[]> content;
186    private JsonObject cachedIndex;
187    private File folder;
188
189    public NpmPackageFolder(String folderName) {
190      super();
191      this.folderName = folderName;
192      if (!minimalMemory) {
193        types = new HashMap<>();
194        content = new HashMap<>();
195      }
196    }
197
198    private String fn(String name) throws IOException {
199      return Utilities.path(folder.getAbsolutePath(), name);
200    }
201
202    public Map<String, List<String>> getTypes() throws JsonException, IOException {
203      if (minimalMemory) {
204        Map<String, List<String>> typeMap = new HashMap<>();
205        readIndex(JsonParser.parseObjectFromFile(fn(".index.json")), typeMap);
206        return typeMap;
207      } else {
208        return types;
209      }
210    }
211
212    public String getFolderName() {
213      return folderName;
214    }
215
216    public String getFolderPath() {
217      return folder == null ? null : folder.getAbsolutePath();
218    }
219    
220    public boolean readIndex(JsonObject index, Map<String, List<String>> typeMap) {
221      if (!index.has("index-version") || (index.asInteger("index-version") != NpmPackageIndexBuilder.CURRENT_INDEX_VERSION)) {
222        return false;
223      }
224      if (!minimalMemory) {
225        this.cachedIndex = index;
226      }
227      for (JsonObject file : index.getJsonObjects("files")) {
228        String type = file.asString("resourceType");
229        String name = file.asString("filename");
230        if (!typeMap.containsKey(type))
231          typeMap.put(type, new ArrayList<>());
232        typeMap.get(type).add(name);
233      }
234      return true;
235    }
236
237    public List<String> listFiles() {
238      List<String> res = new ArrayList<>();
239      if (folder != null) {
240        if (folder.exists()) {
241          for (File f : folder.listFiles()) {
242            if (!f.isDirectory() && !Utilities.existsInList(f.getName(), "package.json", ".index.json", ".index.db", ".oids.json", ".oids.db")) {
243              res.add(f.getName());
244            }
245          }
246        }
247      } else {
248        for (String s : content.keySet()) {
249          if (!Utilities.existsInList(s, "package.json", ".index.json", ".index.db", ".oids.json", ".oids.db")) {
250            res.add(s);
251          }
252        }
253      }
254      Collections.sort(res);
255      return res;
256    }
257
258    public Map<String, byte[]> getContent() {
259      assert !minimalMemory;
260      
261      return content;
262    }
263
264    public byte[] fetchFile(String file) throws FileNotFoundException, IOException {
265      if (folder != null) {
266        File f = ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), file));
267        if (f.exists()) {
268          return TextFile.fileToBytes(f);
269        } else {
270          return null;
271        }
272      } else {
273        return content.get(file);
274      }
275    }
276
277    public ByteProvider getProvider(String file) throws FileNotFoundException, IOException {
278      if (folder != null) {
279        File f = ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), file));
280        if (f.exists()) {
281          return ByteProvider.forFile(f);
282        } else {
283          return null;
284        }
285      } else {
286        return ByteProvider.forBytes(content.get(file));
287      }
288    }
289
290    public boolean hasFile(String file) throws IOException {
291      if (folder != null) {
292        return ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), file)).exists();
293      } else {
294        return content.containsKey(file);
295      }
296
297    }
298
299    public String dump() {
300      return folderName + " ("+ (folder == null ? "null" : folder.toString())+")"+(minimalMemory ? "" : " | "+Boolean.toString(cachedIndex != null)+" | "+content.size()+" | "+types.size());
301    }
302
303    public void removeFile(String n) throws IOException {
304      if (folder != null) {
305        ManagedFileAccess.file(Utilities.path(folder.getAbsolutePath(), n)).delete();
306      } else {
307        content.remove(n);
308      }
309      changedByLoader = true;      
310    }
311
312    public JsonObject index() throws IOException {
313      if (cachedIndex != null) {
314        return cachedIndex;
315      } else if (folder == null) {
316        return null;
317      } else {
318        File ij = ManagedFileAccess.file(fn(".index.json"));
319        if (ij.exists()) {
320          return JsonParser.parseObject(ij);
321        } else {
322          return null;
323        }
324      }
325    }
326    public JsonObject oidIndex() throws IOException {
327      if (folder == null) {
328        return null;
329      } else {
330        File ij = ManagedFileAccess.file(fn(".oids.json"));
331        if (ij.exists()) {
332          return JsonParser.parseObject(ij);
333        } else {
334          return null;
335        }
336      }
337    }
338
339
340  }
341
342  private String path;
343  private JsonObject npm;
344  private Map<String, NpmPackageFolder> folders = new HashMap<>();
345  private boolean changedByLoader; // internal qa only!
346  private Map<String, Object> userData;
347  private boolean minimalMemory;
348  private int size;
349  private boolean warned = false;
350
351  /**
352   * Constructor
353   */
354  private NpmPackage() {
355    super();
356  }
357
358  /**
359   * Factory method that parses a package from an extracted folder
360   */
361  public static NpmPackage fromFolder(String path) throws IOException {
362    return fromFolder(path, true);
363  }
364
365  public static NpmPackage fromFolder(String path, boolean checkIndexed) throws IOException {
366    NpmPackage res = new NpmPackage();
367    res.loadFiles(path, ManagedFileAccess.file(path));
368    if (checkIndexed) {
369     res.checkIndexed(path);
370    }
371    return res;
372  }
373
374  /**
375   * Factory method that parses a package from an extracted folder
376   */
377  public static NpmPackage fromFolderMinimal(String path) throws IOException {
378    return fromFolderMinimal(path, true);
379  }
380
381  public static NpmPackage fromFolderMinimal(String path, boolean checkIndexed) throws IOException {
382    NpmPackage res = new NpmPackage();
383    res.minimalMemory = true;
384    res.loadFiles(path, ManagedFileAccess.file(path));
385    if (checkIndexed) {
386    res.checkIndexed(path);}
387    return res;
388  }
389
390  /**
391   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
392   */
393  public static NpmPackage empty(PackageGenerator thePackageGenerator) {
394    NpmPackage retVal = new NpmPackage();
395    retVal.npm = thePackageGenerator.getRootJsonObject();
396    return retVal;
397  }
398
399  /**
400   * Factory method that starts a new empty package using the given PackageGenerator to create the manifest
401   */
402  public static NpmPackage empty() {
403    NpmPackage retVal = new NpmPackage();
404    return retVal;
405  }
406
407  public Map<String, Object> getUserData() {
408    if (userData == null) {
409      userData = new HashMap<>();
410    }
411    return userData;
412  }
413
414  public void loadFiles(String path, File source, String... exemptions) throws FileNotFoundException, IOException {
415    this.npm = JsonParser.parseObject(TextFile.fileToString(Utilities.path(path, "package", "package.json")));
416    this.path = path;
417    
418    File dir = ManagedFileAccess.file(path);
419    for (File f : dir.listFiles()) {
420      if (!isInternalExemptFile(f) && !Utilities.existsInList(f.getName(), exemptions)) {
421        if (f.isDirectory()) {
422          String d = f.getName();
423          if (!d.equals("package")) {
424            d = Utilities.path("package", d);
425          }
426          File ij = ManagedFileAccess.file(Utilities.path(f.getAbsolutePath(), ".index.json"));
427          NpmPackageFolder folder = this.new NpmPackageFolder(d);
428          folder.folder = f;
429          this.folders.put(d, folder);
430          if (ij.exists() || !minimalMemory) {
431            if (!minimalMemory) {
432              try {
433                if (!ij.exists() || !folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
434                  indexFolder(folder.getFolderName(), folder);
435                }
436              } catch (Exception e) {
437                throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
438              }
439            }
440          }
441          loadSubFolders(dir.getAbsolutePath(), f);
442        } else {
443          NpmPackageFolder folder = this.new NpmPackageFolder(Utilities.path("package", "$root"));
444          folder.folder = dir;
445          this.folders.put(Utilities.path("package", "$root"), folder);        
446        }
447      }
448    }
449  }
450
451  public static boolean isInternalExemptFile(File f) {
452    return Utilities.existsInList(f.getName(), ".git", ".svn", ".DS_Store") || Utilities.existsInList(f.getName(), "package-list.json") ||
453        Utilities.endsWithInList(f.getName(), ".tgz");
454  }
455
456  private void loadSubFolders(String rootPath, File dir) throws IOException {
457    for (File f : dir.listFiles()) {
458      if (f.isDirectory()) {
459        String d = f.getAbsolutePath().substring(rootPath.length()+1);
460        if (!d.startsWith("package")) {
461          d = Utilities.path("package", d);
462        }
463        NpmPackageFolder folder = this.new NpmPackageFolder(d);
464        folder.folder = f;
465        this.folders.put(d, folder);
466        File ij = ManagedFileAccess.file(Utilities.path(f.getAbsolutePath(), ".index.json"));
467        if (ij.exists() || !minimalMemory) {
468          try {
469            if (!ij.exists() || !folder.readIndex(JsonParser.parseObject(ij), folder.getTypes())) {
470              indexFolder(folder.getFolderName(), folder);
471            }
472          } catch (Exception e) {
473            throw new IOException("Error parsing "+ij.getAbsolutePath()+": "+e.getMessage(), e);
474          }
475        }
476        loadSubFolders(rootPath, f);        
477      }
478    }    
479  }
480
481  public static NpmPackage fromFolder(String folder, PackageType defType, String... exemptions) throws IOException {
482    NpmPackage res = new NpmPackage();
483    res.loadFiles(folder, ManagedFileAccess.file(folder), exemptions);
484    if (!res.folders.containsKey("package")) {
485      res.folders.put("package", res.new NpmPackageFolder("package"));
486    }
487    if (!res.folders.get("package").hasFile("package.json") && defType != null) {
488      TextFile.stringToFile("{ \"type\" : \""+defType.getCode()+"\"}", Utilities.path(res.folders.get("package").folder.getAbsolutePath(), "package.json"));
489    }
490    res.npm = JsonParser.parseObject(new String(res.folders.get("package").fetchFile("package.json")));
491    return res;
492  }
493
494  private static final int BUFFER_SIZE = 1024;
495
496  public static @Nonnull NpmPackage fromPackage(InputStream tgz) throws IOException {
497    return fromPackage(tgz, null, false);
498  }
499
500  public static NpmPackage fromPackage(InputStream tgz, String desc) throws IOException {
501    return fromPackage(tgz, desc, false);
502  }
503
504  public static NpmPackage fromPackage(InputStream tgz, String desc, boolean progress) throws IOException {
505    NpmPackage res = new NpmPackage();
506    res.readStream(tgz, desc, progress);
507    return res;
508  }
509  
510  public static NpmPackage extractFromTgz(InputStream tgz, String desc, String tempDir, boolean minimal) throws IOException {
511    Utilities.createDirectory(tempDir);
512
513    int size = 0;
514    
515    GzipCompressorInputStream gzipIn;
516    try {
517      gzipIn = new GzipCompressorInputStream(tgz);
518    } catch (Exception e) {
519      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
520    }
521    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
522      TarArchiveEntry entry;
523
524      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
525        String n = entry.getName();
526        if (n.contains("/..") || n.contains("../")) {
527          throw new RuntimeException("Entry with an illegal name: " + n);
528        }
529        if (entry.isDirectory()) {
530          if (!Utilities.noString(n)) {
531            String dir = n.substring(0, n.length()-1);
532            Utilities.createDirectory(Utilities.path(tempDir, dir));
533          }
534        } else {
535          int count;
536          byte data[] = new byte[BUFFER_SIZE];
537          String filename = Utilities.path(tempDir, n);
538          String folder = Utilities.getDirectoryForFile(filename);
539          Utilities.createDirectory(folder);
540          FileOutputStream fos = ManagedFileAccess.outStream(filename);
541          try (BufferedOutputStream dst = new BufferedOutputStream(fos, BUFFER_SIZE)) {
542            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
543              dst.write(data, 0, count);
544              size = size + count;
545            }
546          }
547          fos.close();
548        }
549      }
550    } 
551    try {
552      NpmPackage npm = NpmPackage.fromFolderMinimal(tempDir);
553      npm.setSize(size);
554      if (!minimal) {
555        npm.checkIndexed(desc);
556      }
557      return npm;
558    } catch (Exception e) {
559      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
560    } 
561  }
562  
563  public void readStream(InputStream tgz, String desc, boolean progress) throws IOException {
564    GzipCompressorInputStream gzipIn;
565    try {
566      gzipIn = new GzipCompressorInputStream(tgz);
567    } catch (Exception e) {
568      throw new IOException("Error reading "+(desc == null ? "package" : desc)+": "+e.getMessage(), e);      
569    }
570    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
571      TarArchiveEntry entry;
572
573      int i = 0;
574      int c = 12;
575      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
576        i++;
577        String n = entry.getName();
578        if (n.contains("..")) {
579          throw new RuntimeException("Entry with an illegal name: " + n);
580        }
581        if (entry.isDirectory()) {
582          String dir = n.substring(0, n.length()-1);
583          if (dir.startsWith("package/")) {
584            dir = dir.substring(8);
585          }
586          folders.put(dir, new NpmPackageFolder(dir));
587        } else {
588          int count;
589          byte data[] = new byte[BUFFER_SIZE];
590          ByteArrayOutputStream fos = new ByteArrayOutputStream();
591          try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
592            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
593              dest.write(data, 0, count);
594            }
595          }
596          fos.close();
597          loadFile(n, fos.toByteArray());
598        }
599        if (progress && i % 50 == 0) {
600          c++;
601          System.out.print(".");
602          if (c == 120) {
603            System.out.println("");
604            System.out.print("  ");
605            c = 2;
606          }
607        }
608      }
609    } 
610    try {
611      npm = JsonParser.parseObject(folders.get("package").fetchFile("package.json"));
612    } catch (Exception e) {
613      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
614    }
615    checkIndexed(desc);
616  }
617
618  public void loadFile(String n, byte[] data) throws IOException {
619    String dir = n.contains("/") ? n.substring(0, n.lastIndexOf("/")) : "$root";
620    if (dir.startsWith("package/")) {
621      dir = dir.substring(8);
622    }
623    n = n.substring(n.lastIndexOf("/")+1);
624    NpmPackageFolder index = folders.get(dir);
625    if (index == null) {
626      index = new NpmPackageFolder(dir);
627      folders.put(dir, index);
628    }
629    index.content.put(n, data);
630  }
631
632  public boolean isIndexed() throws IOException {
633    for (NpmPackageFolder folder : folders.values()) {
634      JsonObject index = folder.index();
635      if (folder.index() == null || index.forceArray("files").size() == 0) {
636        return false;
637      }
638    }
639    return true;
640  }
641
642
643  public void checkIndexed(String desc) throws IOException {
644    for (NpmPackageFolder folder : folders.values()) {
645      JsonObject index = folder.index();
646      if (index == null || index.forceArray("files").size() == 0) {
647        indexFolder(desc, folder);
648      }  
649    }
650  }
651
652
653  /**
654   * Create a package .index.json file for a package folder.
655   * <p>
656   * See <a href="https://hl7.org/fhir/packages.html#2.1.10.4">the FHIR specification</a> for details on .index.json
657   * format and usage.
658   *
659   * @param desc
660   * @param folder
661   * @throws FileNotFoundException
662   * @throws IOException
663   */
664  public void indexFolder(String desc, NpmPackageFolder folder) throws FileNotFoundException, IOException {
665    List<String> remove = new ArrayList<>();
666    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
667    indexer.start(folder.folder != null ? Utilities.path(folder.folder.getAbsolutePath(), ".index.db") : null);
668    for (String n : folder.listFiles()) {
669      if (!indexer.seeFile(n, folder.fetchFile(n))) {
670        remove.add(n);
671      }
672    } 
673    for (String n : remove) {
674      folder.removeFile(n);
675    }
676    String json = indexer.build();
677    
678    try {
679      if (!minimalMemory) {
680        folder.readIndex(JsonParser.parseObject(json), folder.getTypes());
681      }
682      if (folder.folder != null) {
683        TextFile.stringToFile(json, Utilities.path(folder.folder.getAbsolutePath(), ".index.json"));
684      }
685    } catch (Exception e) {
686      TextFile.stringToFile(json, Utilities.path("[tmp]", ".index.json"));
687      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/"+folder.folderName+"/.index.json: "+e.getMessage(), e);
688    }
689  }
690
691
692  public static NpmPackage fromZip(InputStream stream, boolean dropRootFolder, String desc) throws IOException {
693    NpmPackage res = new NpmPackage();
694    ZipInputStream zip = new ZipInputStream(stream);
695    ZipEntry ze;
696    while ((ze = zip.getNextEntry()) != null) {
697      int size;
698      byte[] buffer = new byte[2048];
699
700      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
701      BufferedOutputStream bos = new BufferedOutputStream(bytes, buffer.length);
702
703      while ((size = zip.read(buffer, 0, buffer.length)) != -1) {
704        bos.write(buffer, 0, size);
705      }
706      bos.flush();
707      bos.close();
708      if (bytes.size() > 0) {
709        if (dropRootFolder) {
710          res.loadFile(ze.getName().substring(ze.getName().indexOf("/")+1), bytes.toByteArray());
711        } else {
712          res.loadFile(ze.getName(), bytes.toByteArray());
713        }
714      }
715      zip.closeEntry();
716    }
717    zip.close();         
718    try {
719      res.npm = JsonParser.parseObject(res.folders.get("package").fetchFile("package.json"));
720    } catch (Exception e) {
721      throw new IOException("Error parsing "+(desc == null ? "" : desc+"#")+"package/package.json: "+e.getMessage(), e);
722    }
723    res.checkIndexed(desc);
724    return res;
725  }
726
727
728  /**
729   * Accessing the contents of the package - get a list of files in a subfolder of the package 
730   *
731   * @param folder
732   * @return
733   * @throws IOException 
734   */
735  public List<String> list(String folder) throws IOException {
736    List<String> res = new ArrayList<String>();
737    if (folders.containsKey(folder)) {
738      for (String s : folders.get(folder).listFiles()) {
739        if (!s.startsWith(".")) {
740          res.add(s);
741        }
742      }
743    } else if (folders.containsKey(Utilities.path("package", folder))) {
744      for (String s : folders.get(Utilities.path("package", folder)).listFiles()) {
745        if (!s.startsWith(".")) {
746          res.add(s);
747        }
748      }
749    }
750    return res;
751  }
752
753  public List<String> listResources(String... types) throws IOException {
754    return listResources(Utilities.strings(types));
755  }
756  
757  public List<String> listResources(List<String> types) throws IOException {
758    List<String> res = new ArrayList<String>();
759    NpmPackageFolder folder = folders.get("package");
760    if (types.size() == 0) {
761      for (String s : folder.types.keySet()) {
762        if (folder.types.containsKey(s)) {
763          res.addAll(folder.types.get(s));
764        }
765      }
766    } else {
767      for (String s : types) {
768        if (folder.types.containsKey(s)) {
769          res.addAll(folder.types.get(s));
770        }
771      }
772    }
773    Collections.sort(res);
774    return res;
775  }
776
777  public List<PackageResourceInformation> listIndexedResources(String... types) throws IOException {
778    return listIndexedResources(Utilities.strings(types));
779  }
780  
781  public List<PackageResourceInformation> listIndexedResources(List<String> types) throws IOException {
782    List<PackageResourceInformation> res = new ArrayList<PackageResourceInformation>();
783    for (NpmPackageFolder folder : folders.values()) {
784      JsonObject index = folder.index();
785      if (index != null) {
786        for (JsonObject fi : index.getJsonObjects("files")) {
787          if (Utilities.existsInList(fi.asString("resourceType"), types) || types.isEmpty()) {
788            res.add(new PackageResourceInformation(folder.folder == null ? "@"+folder.getFolderName() : folder.folder.getAbsolutePath(), fi));
789          }
790        }
791      }
792    } 
793    //    Collections.sort(res, new PackageResourceInformationSorter());
794    return res;
795  }
796
797  /**
798   * use the name from listResources()
799   * 
800   * @param file
801   * @return
802   * @throws IOException
803   */
804  public InputStream loadResource(String file) throws IOException {
805    NpmPackageFolder folder = folders.get("package");
806    return new ByteArrayInputStream(folder.fetchFile(file));
807  }
808
809  /**
810   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
811   * 
812   * @param canonical - the canonical URL of the resource (exact match only)
813   * @return null if it is not found
814   * @throws IOException
815   */
816  public InputStream loadByCanonical(String canonical) throws IOException {
817    return loadByCanonicalVersion("package", canonical, null);    
818  }
819  
820  /**
821   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
822   * 
823   * @param folder - one of the folders in the package (main folder is "package")
824   * @param canonical - the canonical URL of the resource (exact match only)
825   * @return null if it is not found
826   * @throws IOException
827   */
828  public InputStream loadByCanonical(String folder, String canonical) throws IOException {
829    return loadByCanonicalVersion(folder, canonical, null);    
830  }
831    
832  /**
833   * get a stream that contains the contents of a resource in the base folder, by it's canonical URL
834   * 
835   * @param canonical - the canonical URL of the resource (exact match only)
836   * @param version - the specified version (or null if the most recent)
837   * 
838   * @return null if it is not found
839   * @throws IOException
840   */
841  public InputStream loadByCanonicalVersion(String canonical, String version) throws IOException {
842    return loadByCanonicalVersion("package", canonical, version);
843  }
844  
845  /**
846   * get a stream that contains the contents of a resource in the nominated folder, by it's canonical URL
847   * 
848   * @param folder - one of the folders in the package (main folder is "package")
849   * @param canonical - the canonical URL of the resource (exact match only)
850   * @param version - the specified version (or null if the most recent)
851   * 
852   * @return null if it is not found
853   * @throws IOException
854   */
855  public InputStream loadByCanonicalVersion(String folder, String canonical, String version) throws IOException {
856    NpmPackageFolder f = folders.get(folder);
857    List<JsonObject> matches = new ArrayList<>();
858    for (JsonObject file : f.index().getJsonObjects("files")) {
859      if (canonical.equals(file.asString("url"))) {
860        if (version != null && version.equals(file.asString("version"))) {
861          return load("package", file.asString("filename"));
862        } else if (version == null) {
863          matches.add(file);
864        }
865      }
866      if (matches.size() > 0) {
867        if (matches.size() == 1) {
868          return load("package", matches.get(0).asString("filename"));          
869        } else {
870          Collections.sort(matches, new IndexVersionSorter());
871          return load("package", matches.get(matches.size()-1).asString("filename"));          
872        }
873      }
874    }
875    return null;        
876  }
877    
878  /**
879   * get a stream that contains the contents of one of the files in the base package
880   * 
881   * @param file
882   * @return
883   * @throws IOException
884   */
885  public InputStream load(String file) throws IOException {
886    return load("package", file);
887  }
888  
889  /**
890   * get a stream that contains the contents of one of the files in a folder
891   * 
892   * @param folder
893   * @param file
894   * @return
895   * @throws IOException
896   */
897  public InputStream load(String folder, String file) throws IOException {
898    NpmPackageFolder f = folders.get(folder);
899    if (f == null) {
900      f = folders.get(Utilities.path("package", folder));
901    }
902    if (f != null && f.hasFile(file)) {
903      return new ByteArrayInputStream(f.fetchFile(file));
904    } else {
905      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
906    }
907  }
908
909  /**
910   * get a stream that contains the contents of one of the files in a folder
911   * 
912   * @param folder
913   * @param file
914   * @return
915   * @throws IOException
916   */
917  public ByteProvider getProvider(String folder, String file) throws IOException {
918    NpmPackageFolder f = folders.get(folder);
919    if (f == null) {
920      f = folders.get(Utilities.path("package", folder));
921    }
922    if (f != null && f.hasFile(file)) {
923      return f.getProvider(file);
924    } else {
925      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
926    }
927  }
928
929  
930  public boolean hasFile(String folder, String file) throws IOException {
931    NpmPackageFolder f = folders.get(folder);
932    if (f == null) {
933      f = folders.get(Utilities.path("package", folder));
934    }
935    return f != null && f.hasFile(file);
936  }
937
938
939  /**
940   * Handle to the package json file
941   * 
942   * @return
943   */
944  public JsonObject getNpm() {
945    return npm;
946  }
947
948  /**
949   *  should only use this when npm == null
950   * 
951   * @param npm
952   */
953  public void setNpm(JsonObject npm) {
954    this.npm = npm;
955  }
956
957  /**
958   * convenience method for getting the package name
959   * @return
960   */
961  public String name() {
962    return npm.asString("name");
963  }
964
965  /**
966   * convenience method for getting the package id (which in NPM language is the same as the name)
967   * @return
968   */
969  public String id() {
970    return npm.asString("name");
971  }
972
973  public String date() {
974    return npm.asString("date");
975  }
976
977  public String canonical() {
978    return npm.asString("canonical");
979  }
980
981  /**
982   * convenience method for getting the package version
983   * @return
984   */
985  public String version() {
986    return npm.asString("version");
987  }
988
989  /**
990   * convenience method for getting the package fhir version
991   * @return
992   */
993  public String fhirVersion() {
994    if ("hl7.fhir.core".equals(npm.asString("name")))
995      return npm.asString("version");
996    else if (
997        Utilities.existsInList(npm.asString("type"), "fhir.core", "fhir.examples") &&
998        Utilities.startsWithInList( npm.asString("name"), "hl7.fhir.r2.", "hl7.fhir.r2b.", "hl7.fhir.r3.", 
999             "hl7.fhir.r4.", "hl7.fhir.r4b.", "hl7.fhir.r5.")) {
1000      return npm.asString("version");
1001    } else {
1002      JsonObject dep = null;
1003      if (npm.hasObject("dependencies")) {
1004        dep = npm.getJsonObject("dependencies");
1005        if (dep != null) {
1006          for (JsonProperty e : dep.getProperties()) {
1007            if (Utilities.existsInList(e.getName(), "hl7.fhir.r2.core", "hl7.fhir.r2b.core", "hl7.fhir.r3.core", "hl7.fhir.r4.core"))
1008              return e.getValue().asString();
1009            if (Utilities.existsInList(e.getName(), "hl7.fhir.core")) // while all packages are updated
1010              return e.getValue().asString();
1011          }
1012        }
1013      }
1014      if (npm.hasArray("fhirVersions")) {
1015        JsonArray e = npm.getJsonArray("fhirVersions");
1016        if (e.size() > 0) {
1017          return e.getItems().get(0).asString();
1018        }
1019      }
1020      if (dep != null) {
1021        // legacy simplifier support:
1022        if (dep.has("simplifier.core.r4"))
1023          return "4.0";
1024        if (dep.has("simplifier.core.r3"))
1025          return "3.0";
1026        if (dep.has("simplifier.core.r2"))
1027          return "2.0";
1028      }
1029      throw new FHIRException("no core dependency or FHIR Version found in the Package definition");
1030    }
1031  }
1032
1033  public String summary() {
1034    if (path != null)
1035      return path;
1036    else
1037      return "memory";
1038  }
1039
1040  public boolean isType(PackageType template) {
1041    return template.getCode().equals(type()) || template.getOldCode().equals(type()) ;
1042  }
1043
1044  public String type() {
1045    return npm.asString("type");
1046  }
1047
1048  public String description() {
1049    return npm.asString("description");
1050  }
1051
1052  public String getPath() {
1053    return path;
1054  }
1055
1056  public List<String> dependencies() {
1057    List<String> res = new ArrayList<>();
1058    if (npm.has("dependencies")) {
1059      for (JsonProperty e : npm.getJsonObject("dependencies").getProperties()) {
1060        res.add(e.getName()+"#"+e.getValue().asString());
1061      }
1062    }
1063    return res;
1064  }
1065
1066  public String homepage() {
1067    return npm.asString("homepage");
1068  }
1069
1070  public String url() {
1071    return npm.asString("url");
1072  }
1073
1074
1075  public String title() {
1076    return npm.asString("title");
1077  }
1078
1079  public String toolsVersion() {
1080    return npm.asString("tools-version");
1081  }
1082
1083  public String license() {
1084    return npm.asString("license");
1085  }
1086
1087  //  /**
1088  //   * only for use by the package manager itself
1089  //   * 
1090  //   * @param path
1091  //   */
1092  //  public void setPath(String path) {
1093  //    this.path = path;
1094  //  }
1095
1096  public String getWebLocation() {
1097    if (npm.hasPrimitive("url")) {
1098      return PackageHacker.fixPackageUrl(npm.asString("url"));
1099    } else {
1100      return npm.asString("canonical");
1101    }
1102  }
1103
1104  public InputStream loadResource(String type, String id) throws IOException {
1105    NpmPackageFolder f = folders.get("package");
1106    JsonArray files = f.index().getJsonArray("files");
1107    for (JsonElement e : files.getItems()) {
1108      JsonObject i = (JsonObject) e;
1109      if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
1110        return load("package", i.asString("filename"));
1111      }
1112    }
1113    return null;
1114  }
1115
1116  public InputStream loadExampleResource(String type, String id) throws IOException {
1117    NpmPackageFolder f = folders.get("example");
1118    if (f == null) {
1119      f = folders.get("package/example");      
1120    }
1121    if (f != null) {
1122      JsonArray files = f.index().getJsonArray("files");
1123      for (JsonElement e : files.getItems()) {
1124        JsonObject i = (JsonObject) e;
1125        if (type.equals(i.asString("resourceType")) && id.equals(i.asString("id"))) {
1126          return load("example", i.asString("filename"));
1127        }
1128      }
1129    }
1130    return null;
1131  }
1132
1133  /** special case when playing around inside the package **/
1134  public Map<String, NpmPackageFolder> getFolders() {
1135    return folders;
1136  }
1137
1138  public void save(File directory) throws IOException {
1139    assert !minimalMemory;
1140    File dir = ManagedFileAccess.file(Utilities.path(directory.getAbsolutePath(), name()));
1141    if (!dir.exists()) {
1142      Utilities.createDirectory(dir.getAbsolutePath());
1143    } else {
1144      Utilities.clearDirectory(dir.getAbsolutePath());
1145    }
1146    
1147    for (NpmPackageFolder folder : folders.values()) {
1148      String n = folder.folderName;
1149
1150      File pd = ManagedFileAccess.file(Utilities.path(dir.getAbsolutePath(), n));
1151      if (!pd.exists()) {
1152        Utilities.createDirectory(pd.getAbsolutePath());
1153      }
1154      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
1155      indexer.start(Utilities.path(dir.getAbsolutePath(), n, ".index.db"));
1156      for (String s : folder.content.keySet()) {
1157        byte[] b = folder.content.get(s);
1158        indexer.seeFile(s, b);
1159        if (!s.equals(".index.json") && !s.equals("package.json")) {
1160          TextFile.bytesToFile(b, Utilities.path(dir.getAbsolutePath(), n, s));
1161        }
1162      }
1163      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
1164      TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), n, ".index.json"));
1165    }
1166    byte[] cnt = TextFile.stringToBytes(JsonParser.compose(npm, true));
1167    TextFile.bytesToFile(cnt, Utilities.path(dir.getAbsolutePath(), "package", "package.json"));
1168  }
1169  
1170  public void save(OutputStream stream) throws IOException {
1171    assert !minimalMemory;
1172    TarArchiveOutputStream tar;
1173//    ByteArrayOutputStream OutputStream;
1174//    BufferedOutputStream bufferedOutputStream;
1175    GzipCompressorOutputStream gzipOutputStream;
1176
1177//    OutputStream = new ByteArrayOutputStream();
1178//    bufferedOutputStream = new BufferedOutputStream(OutputStream);
1179    GzipParameters gp = new GzipParameters();
1180    gp.setCompressionLevel(Deflater.BEST_COMPRESSION);
1181    gzipOutputStream = new GzipCompressorOutputStream(stream, gp);
1182    tar = new TarArchiveOutputStream(gzipOutputStream);
1183
1184
1185    for (NpmPackageFolder folder : folders.values()) {
1186      String n = folder.folderName;
1187      if (!"package".equals(n) && !(n.startsWith("package/") || n.startsWith("package\\"))) {
1188        n = "package/"+n;
1189      }
1190      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
1191      String filename = Utilities.path("[tmp]", "tmp-"+UUID.randomUUID().toString()+".db");
1192      indexer.start(filename);
1193      for (String s : folder.content.keySet()) {
1194        byte[] b = folder.content.get(s);
1195        String name = n+"/"+s;
1196        if (b == null) {
1197          System.out.println(name+" is null");
1198        } else {
1199          indexer.seeFile(s, b);
1200          if (!s.equals(".index.json") && !s.equals(".index.db") && !s.equals("package.json")) {
1201            TarArchiveEntry entry = new TarArchiveEntry(name);
1202            entry.setSize(b.length);
1203            tar.putArchiveEntry(entry);
1204            tar.write(b);
1205            tar.closeArchiveEntry();
1206          }
1207        }
1208      }
1209      byte[] cnt = indexer.build().getBytes(StandardCharsets.UTF_8);
1210      TarArchiveEntry entry = new TarArchiveEntry(n+"/.index.json");
1211      entry.setSize(cnt.length);
1212      tar.putArchiveEntry(entry);
1213      tar.write(cnt);
1214      tar.closeArchiveEntry();
1215      var file = ManagedFileAccess.file(filename);
1216      if (file.exists()) {
1217        cnt = TextFile.fileToBytes(file);
1218        file.delete();
1219        entry = new TarArchiveEntry(n+"/.index.db");
1220        entry.setSize(cnt.length);
1221        tar.putArchiveEntry(entry);
1222        tar.write(cnt);
1223        tar.closeArchiveEntry();
1224      }
1225    }
1226    byte[] cnt = TextFile.stringToBytes(JsonParser.compose(npm, true));
1227    TarArchiveEntry entry = new TarArchiveEntry("package/package.json");
1228    entry.setSize(cnt.length);
1229    tar.putArchiveEntry(entry);
1230    tar.write(cnt);
1231    tar.closeArchiveEntry();
1232
1233    tar.finish();
1234    tar.close();
1235    gzipOutputStream.close();
1236//    bufferedOutputStream.close();
1237//    OutputStream.close();
1238//    byte[] b = OutputStream.toByteArray();
1239//    stream.write(b);
1240  }
1241
1242  /**
1243   * Keys are resource type names, values are filenames
1244   */
1245  public Map<String, List<String>> getTypes() {
1246    return folders.get("package").types;
1247  }
1248
1249  public String fhirVersionList() {
1250    if (npm.has("fhirVersions")) {
1251      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
1252      if (npm.hasArray("fhirVersions")) {
1253        for (String n : npm.getJsonArray("fhirVersions").asStrings()) {
1254          b.append(n);
1255        }
1256      }
1257      if (npm.hasPrimitive("fhirVersions")) {
1258        b.append(npm.asString("fhirVersions"));
1259      }
1260      return b.toString();
1261    } else
1262      return "";
1263  }
1264
1265  public String dependencySummary() {
1266    if (npm.has("dependencies")) {
1267      CommaSeparatedStringBuilder b = new CommaSeparatedStringBuilder();
1268      for (JsonProperty e : npm.getJsonObject("dependencies").getProperties()) {
1269        b.append(e.getName()+"#"+e.getValue().asString());
1270      }
1271      return b.toString();
1272    } else
1273      return "";
1274  }
1275
1276  public void unPack(String dir) throws IOException {
1277    unPack (dir, false, new ArrayList<>());
1278  }
1279
1280  public void unPackWithAppend(String dir, List<String> files) throws IOException {
1281    unPack (dir, true, files);
1282  }
1283
1284  public void unPack(String dir, boolean withAppend) throws IOException {
1285    unPack (dir, withAppend, new ArrayList<>());
1286  }
1287  
1288  public void unPack(String dir, boolean withAppend, List<String> files) throws IOException {
1289    assert !minimalMemory;
1290    
1291    for (NpmPackageFolder folder : folders.values()) {
1292      String dn = folder.getFolderName();
1293      if (!dn.equals("package") && (dn.startsWith("package/") || dn.startsWith("package\\"))) {
1294        dn = dn.substring(8);
1295      }
1296      if (dn.equals("$root")) {
1297        dn = dir;
1298      } else {
1299         dn = Utilities.path(dir, dn);
1300      }
1301      Utilities.createDirectory(dn);
1302      for (String s : folder.listFiles()) {
1303        String fn = Utilities.path(dn, s);
1304        File f = ManagedFileAccess.file(fn);
1305        if (withAppend && f.getName().startsWith("_append.")) {
1306          String appendFn = Utilities.path(dn, s.substring(8));
1307          f = ManagedFileAccess.file(appendFn);
1308          files.add(f.getAbsolutePath());
1309          if (f.exists())
1310            TextFile.appendBytesToFile(folder.fetchFile(s), appendFn);        
1311          else
1312            TextFile.bytesToFile(folder.fetchFile(s), appendFn);        
1313        } else
1314          files.add(f.getAbsolutePath());
1315          TextFile.bytesToFile(folder.fetchFile(s), fn);
1316      }      
1317    }
1318  }
1319
1320  public void debugDump(String purpose) {
1321//    System.out.println("Debug Dump of Package for '"+purpose+"'. Path = "+path);
1322//    System.out.println("  npm = "+name()+"#"+version()+", canonical = "+canonical());
1323//    System.out.println("  folders = "+folders.size());
1324//    for (String s : sorted(folders.keySet())) {
1325//      NpmPackageFolder folder = folders.get(s);
1326//      System.out.println("    "+folder.dump());
1327//    }
1328  }
1329
1330  private List<String> sorted(Set<String> keys) {
1331    List<String> res = new ArrayList<String>();
1332    res.addAll(keys);
1333    Collections.sort(res);
1334    return res ;
1335  }
1336
1337  public void clearFolder(String folderName) {
1338    NpmPackageFolder folder = folders.get(folderName);
1339    folder.content.clear();
1340    folder.types.clear();    
1341  }
1342
1343  public void deleteFolder(String folderName) {
1344    folders.remove(folderName);
1345  }
1346
1347  public void addFile(String folderName, String name, byte[] cnt, String type) {
1348    assert !minimalMemory;
1349    
1350    if (!folders.containsKey(folderName)) {
1351      folders.put(folderName, new NpmPackageFolder(folderName));
1352    }
1353    NpmPackageFolder folder = folders.get(folderName);
1354    folder.content.put(name, cnt);
1355    if (!folder.types.containsKey(type))
1356      folder.types.put(type, new ArrayList<>());
1357    folder.types.get(type).add(name);
1358    if ("package".equals(folderName) && "package.json".equals(name)) {
1359      try {
1360        npm = JsonParser.parseObject(cnt);
1361      } catch (IOException e) {
1362      }
1363    }
1364  }
1365
1366  public void loadAllFiles() throws IOException {
1367    for (String folder : folders.keySet()) {
1368      NpmPackageFolder pf = folders.get(folder);
1369      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1370      File file = ManagedFileAccess.file(p);
1371      if (file.exists()) {
1372        for (File f : file.listFiles()) {
1373          if (!f.isDirectory() && !isInternalExemptFile(f)) {
1374            pf.getContent().put(f.getName(), TextFile.fileToBytes(f));
1375          }
1376        }
1377      }
1378    }
1379  }
1380
1381  public void loadAllFiles(ITransformingLoader loader) throws IOException {
1382    for (String folder : folders.keySet()) {
1383      NpmPackageFolder pf = folders.get(folder);
1384      String p = folder.contains("$") ? path : Utilities.path(path, folder);
1385      for (File f : ManagedFileAccess.file(p).listFiles()) {
1386        if (!f.isDirectory() && !isInternalExemptFile(f)) {
1387          pf.getContent().put(f.getName(), loader.load(f));
1388        }
1389      }
1390    }
1391  }
1392
1393  public boolean isChangedByLoader() {
1394    return changedByLoader;
1395  }
1396
1397  public boolean isCore() {
1398    return Utilities.existsInList(npm.asString("type"), "fhir.core", "Core");
1399  }
1400
1401  public boolean isCoreExamples() {
1402    return name().startsWith("hl7.fhir.r") && name().endsWith(".examples");
1403  }
1404  
1405  
1406  public boolean isTx() {
1407    return npm.asString("name").startsWith("hl7.terminology");
1408  }
1409
1410  public boolean hasCanonical(String url) throws IOException {
1411    if (url == null) {
1412      return false;
1413    }
1414    String u = url.contains("|") ?  url.substring(0, url.indexOf("|")) : url;
1415    String v = url.contains("|") ?  url.substring(url.indexOf("|")+1) : null;
1416    NpmPackageFolder folder = folders.get("package");
1417    if (folder != null) {
1418      for (JsonObject o : folder.index().getJsonObjects("files")) {
1419        if (u.equals(o.asString("url"))) {
1420          if (v == null || v.equals(o.asString("version"))) {
1421            return true;
1422          }
1423        }
1424      }
1425    }
1426    return false;
1427  }
1428
1429  public boolean canLazyLoad() throws IOException {
1430    for (NpmPackageFolder folder : folders.values()) {
1431      if (folder.folder == null) {        
1432        return false;
1433      }
1434    }
1435    if (Utilities.existsInList(name(), "fhir.test.data.r2", "fhir.test.data.r3", "fhir.test.data.r4", "fhir.tx.support.r2", "fhir.tx.support.r3", "fhir.tx.support.r4", "us.nlm.vsac")) {
1436      return true;
1437    }
1438    if (npm.asBoolean("lazy-load")) {
1439      return true;
1440    }
1441    if (!hasFile("other", "spec.internals") && folders.get("package").cachedIndex == null) {
1442      return false;
1443    }
1444    return true;
1445  }
1446
1447  public boolean isNotForPublication() {
1448    return npm.asBoolean("notForPublication");
1449 }
1450
1451  public InputStream load(PackageResourceInformation p) throws IOException {
1452    if (p.filename.startsWith("@")) {
1453      String[] pl = p.filename.substring(1).split("\\/");
1454      return new ByteArrayInputStream(folders.get(pl[0]).content.get(pl[1]));
1455    } else {
1456      return ManagedFileAccess.inStream(p.filename);
1457    }
1458  }
1459
1460  public Date dateAsDate() {
1461    try {
1462      String d = date();
1463      if (d == null) {
1464        switch (name()) {
1465        case "hl7.fhir.r2.core":  d = "20151024000000"; break;
1466        case "hl7.fhir.r2b.core": d = "20160330000000"; break;
1467        case "hl7.fhir.r3.core":  d = "20191024000000"; break;
1468        case "hl7.fhir.r4.core":  d = "20191030000000"; break;
1469        case "hl7.fhir.r4b.core": d = "202112200000000"; break;
1470        case "hl7.fhir.r5.core":  d = "20211219000000"; break;
1471        default:
1472          return new Date();
1473        }
1474      }
1475      return new SimpleDateFormat("yyyyMMddHHmmss").parse(d);
1476    } catch (ParseException e) {
1477      // this really really shouldn't happen
1478      return new Date();
1479    }
1480  }
1481
1482  public static NpmPackage fromUrl(String source) throws IOException {
1483    HTTPResult res = ManagedWebAccess.get(source+"?nocache=" + System.currentTimeMillis());
1484    res.checkThrowException();
1485    return fromPackage(new ByteArrayInputStream(res.getContent()));
1486  }
1487
1488  @Override
1489  public String toString() {
1490    return "NpmPackage "+name()+"#"+version()+" [path=" + path + "]";
1491  }
1492
1493  public String getFilePath(String d) throws IOException {
1494    return Utilities.path(path, "package", d);
1495  }
1496
1497  public boolean isMinimalMemory() {
1498    return minimalMemory;
1499  }
1500
1501  public int getSize() {
1502    return size;
1503  }
1504
1505  public void setSize(int size) {
1506    this.size = size;
1507  }
1508
1509  public boolean isWarned() {
1510    return warned;
1511  }
1512
1513  public void setWarned(boolean warned) {
1514    this.warned = warned;
1515  }
1516
1517  public String vid() {
1518    return id()+"#"+version();
1519  }
1520
1521  
1522}