001package org.hl7.fhir.utilities.cache;
002
003/*-
004 * #%L
005 * org.hl7.fhir.utilities
006 * %%
007 * Copyright (C) 2014 - 2019 Health Level 7
008 * %%
009 * Licensed under the Apache License, Version 2.0 (the "License");
010 * you may not use this file except in compliance with the License.
011 * You may obtain a copy of the License at
012 * 
013 *      http://www.apache.org/licenses/LICENSE-2.0
014 * 
015 * Unless required by applicable law or agreed to in writing, software
016 * distributed under the License is distributed on an "AS IS" BASIS,
017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
018 * See the License for the specific language governing permissions and
019 * limitations under the License.
020 * #L%
021 */
022
023
024import java.io.BufferedOutputStream;
025import java.io.ByteArrayInputStream;
026import java.io.ByteArrayOutputStream;
027import java.io.File;
028import java.io.FileInputStream;
029import java.io.FileNotFoundException;
030import java.io.IOException;
031import java.io.InputStream;
032import java.io.OutputStream;
033import java.nio.charset.Charset;
034import java.util.ArrayList;
035import java.util.Collections;
036import java.util.HashMap;
037import java.util.List;
038import java.util.Map;
039import java.util.Map.Entry;
040import java.util.zip.ZipEntry;
041import java.util.zip.ZipInputStream;
042
043import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
044import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
045import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
046import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
047import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
048import org.apache.commons.io.FileUtils;
049import org.apache.commons.io.IOUtils;
050import org.hl7.fhir.exceptions.FHIRException;
051import org.hl7.fhir.utilities.IniFile;
052import org.hl7.fhir.utilities.TextFile;
053import org.hl7.fhir.utilities.Utilities;
054import org.hl7.fhir.utilities.cache.PackageCacheManager.PackageEntry;
055import org.hl7.fhir.utilities.cache.PackageGenerator.PackageType;
056import org.hl7.fhir.utilities.json.JsonTrackingParser;
057
058import com.google.common.base.Charsets;
059import com.google.gson.GsonBuilder;
060import com.google.gson.JsonArray;
061import com.google.gson.JsonElement;
062import com.google.gson.JsonObject;
063
064/**
065 * info and loader for a package 
066 * 
067 * Packages may exist on disk in the cache, or purely in memory when they are loaded on the fly
068 * 
069 * Packages are contained in subfolders (see the package spec). The FHIR resources will be in "package"
070 * 
071 * @author Grahame Grieve
072 *
073 */
074public class NpmPackage {
075
076  private String path;
077  private List<String> folders = new ArrayList<String>();
078  private Map<String, byte[]> content = new HashMap<String, byte[]>();
079  private JsonObject npm;
080  private IniFile cache;
081  private Map<String, List<String>> types = new HashMap<>();
082  private Map<String, String> urls = new HashMap<>();
083
084  public NpmPackage(JsonObject npm, Map<String, byte[]> content, List<String> folders) {
085    this.path = null;
086    this.content = content;
087    this.npm = npm;
088    this.folders = folders;
089  }
090
091  public NpmPackage(String path) throws IOException {
092    this.path = path;
093    if (path != null) {
094      for (String f : sorted(new File(path).list())) {
095        if (new File(Utilities.path(path, f)).isDirectory()) {
096          folders.add(f); 
097        }
098      }
099      npm = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(Utilities.path(path, "package", "package.json")));
100      File ji = new File(Utilities.path(path, "package", ".index.json"));
101      if (ji.exists()) {
102        readIndexFile((JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(ji)));
103      }
104      cache = new IniFile(Utilities.path(path, "cache.ini"));        
105    }
106  }
107
108  private void readIndexFile(JsonObject index) throws IOException {
109    for (JsonElement e : index.getAsJsonArray("files")) {
110      JsonObject file = (JsonObject) e;
111      String type = file.get("resourceType").getAsString();
112      String name = file.get("filename").getAsString();
113      String url = file.has("url") ? file.get("url").getAsString() : null;
114      String version = file.has("version") ? file.get("version").getAsString() : null;
115      if (!types.containsKey(type))
116        types.put(type, new ArrayList<>());
117      types.get(type).add(name);
118      if (version != null)
119        url = url + "|" + version;
120      urls.put(url, name);
121    }
122  }
123
124  private List<String> sorted(String[] keys) {
125    List<String> names = new ArrayList<String>();
126    for (String s : keys)
127      names.add(s);
128    Collections.sort(names);
129    return names;
130  }
131
132  private static final int BUFFER_SIZE = 1024;
133
134  public static NpmPackage fromPackage(InputStream tgz) throws IOException {
135    return fromPackage(tgz, false);
136  }
137
138  public static NpmPackage fromPackage(InputStream tgz, boolean progress) throws IOException {
139    NpmPackage res = new NpmPackage(null);
140    boolean hasIndex = false;
141    GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(tgz);
142    try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) {
143      TarArchiveEntry entry;
144
145      int i = 0;
146      int c = 12;
147      while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) {
148        i++;
149        if (entry.isDirectory()) {
150          res.folders.add(entry.getName());
151        } else {
152          hasIndex = hasIndex || "package/.index.json".equals(entry.getName());
153          int count;
154          byte data[] = new byte[BUFFER_SIZE];
155
156          ByteArrayOutputStream fos = new ByteArrayOutputStream();
157          try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) {
158            while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) {
159              dest.write(data, 0, count);
160            }
161          }
162          fos.close();
163          res.content.put(entry.getName(), fos.toByteArray());
164        }
165        i++;
166        if (progress && i % 50 == 0) {
167          c++;
168          System.out.print(".");
169          if (c == 120) {
170            System.out.println("");
171            System.out.print("  ");
172            c = 2;
173          }
174        }    
175      }
176    }
177    if (!hasIndex) {
178      NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
179      indexer.start();
180      for (String n : res.getContent().keySet()) {
181        if (n.startsWith("package/")) {
182          indexer.seeFile(n.substring(8), res.content.get(n));
183        }          
184      }
185      res.content.put("package/.index.json", indexer.build().getBytes(Charsets.UTF_8));
186    }
187    res.npm = JsonTrackingParser.parseJson(res.content.get("package/package.json"));
188    res.readIndexFile((JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/.index.json"))));
189    return res;
190  }
191
192
193  public static NpmPackage fromZip(InputStream stream, boolean dropRootFolder) throws IOException {
194    NpmPackage res = new NpmPackage(null);
195    ZipInputStream zip = new ZipInputStream(stream);
196    ZipEntry ze;
197    while ((ze = zip.getNextEntry()) != null) {
198      int size;
199      byte[] buffer = new byte[2048];
200
201      ByteArrayOutputStream bytes = new ByteArrayOutputStream();
202      BufferedOutputStream bos = new BufferedOutputStream(bytes, buffer.length);
203
204      while ((size = zip.read(buffer, 0, buffer.length)) != -1) {
205        bos.write(buffer, 0, size);
206      }
207      bos.flush();
208      bos.close();
209      if (bytes.size() > 0)
210        if (dropRootFolder)
211          res.content.put(ze.getName().substring(ze.getName().indexOf("/")+1), bytes.toByteArray());
212        else
213          res.content.put(ze.getName(), bytes.toByteArray());
214
215      zip.closeEntry();
216    }
217    zip.close();         
218    res.npm = (JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/package.json")));
219    if (res.content.containsKey("package/.index.json"))
220      res.readIndexFile((JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/.index.json"))));
221    return res;
222  }
223
224
225  /**
226   * Accessing the contents of the package - get a list of files in a subfolder of the package 
227   *
228   * @param folder
229   * @return
230   * @throws IOException 
231   */
232  public List<String> list(String folder) throws IOException {
233    List<String> res = new ArrayList<String>();
234    if (path != null) {
235      File f = new File(Utilities.path(path, folder));
236      if (f.exists() && f.isDirectory())
237        for (String s : f.list())
238          res.add(s);
239    } else {
240      for (String s : content.keySet()) {
241        if (s.startsWith(folder+"/") && !s.substring(folder.length()+2).contains("/"))
242          res.add(s.substring(folder.length()+1));
243      }
244    }
245    return res;
246  }
247
248  public List<String> listResources(String... types) throws IOException {
249    List<String> res = new ArrayList<String>();
250    if (this.types.size() > 0) {
251      for (String s : types) {
252        if (this.types.containsKey(s))
253          res.addAll(this.types.get(s));
254      }
255    } else {
256      List<String> files = list("package");
257      for (String s : files) {
258        String[] n = s.split("\\-");
259        if (Utilities.existsInList(n[0], types))
260          res.add(s);
261      }
262    }
263    return res;
264  }
265
266  /** 
267   * Copies all the files in the package folder [folder] to the nominated dest, 
268   * and returns a list of all the file names copied
269   *  
270   * @param folder
271   * @return
272   * @throws IOException 
273   */
274  public List<String> copyTo(String folder, String dest) throws IOException {
275    List<String> res = new ArrayList<String>();
276    if (path != null) {
277      copyToDest(Utilities.path(path, folder), Utilities.path(path, folder), dest, res);
278    } else {
279      for (Entry<String, byte[]> e : content.entrySet()) {
280        if (e.getKey().startsWith(folder+"/")) {
281          String s = e.getKey().substring(folder.length()+1);
282          res.add(s);
283          String dst = Utilities.path(dest, s);
284          String dstDir = Utilities.getDirectoryForFile(dst);
285          Utilities.createDirectory(dstDir);
286          TextFile.bytesToFile(e.getValue(), dst);
287        }
288      }
289    }
290    return res;
291  }
292
293  private void copyToDest(String base, String folder, String dest, List<String> res) throws IOException {
294    for (File f : new File(folder).listFiles()) {
295      if (f.isDirectory()) {
296        copyToDest(base, f.getAbsolutePath(),  Utilities.path(dest, f.getName()), res);
297      } else {
298        String dst = Utilities.path(dest, f.getName());
299        FileUtils.copyFile(f, new File(dst), true);
300        res.add(f.getAbsolutePath().substring(base.length()+1));
301      }
302    }
303  }
304
305  /**
306   * get a stream that contains the contents of one of the files in a folder
307   * 
308   * @param folder
309   * @param file
310   * @return
311   * @throws IOException
312   */
313  public InputStream load(String folder, String file) throws IOException {
314    if (content.containsKey(folder+"/"+file))
315      return new ByteArrayInputStream(content.get(folder+"/"+file));
316    else {
317      File f = new File(Utilities.path(path, folder, file));
318      if (f.exists())
319        return new FileInputStream(f);
320      throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name());
321    }
322  }
323
324  /**
325   * Handle to the package json file
326   * 
327   * @return
328   */
329  public JsonObject getNpm() {
330    return npm;
331  }
332
333  /**
334   * convenience method for getting the package name
335   * @return
336   */
337  public String name() {
338    return npm.get("name").getAsString();
339  }
340
341  public String date() {
342    return npm.has("date") ? npm.get("date").getAsString() : null;
343  }
344
345  public String canonical() {
346    return npm.has("canonical") ? npm.get("canonical").getAsString() : null;
347  }
348
349  /**
350   * convenience method for getting the package version
351   * @return
352   */
353  public String version() {
354    return npm.get("version").getAsString();
355  }
356
357  /**
358   * convenience method for getting the package fhir version
359   * @return
360   */
361  public String fhirVersion() {
362    if ("hl7.fhir.core".equals(npm.get("name").getAsString()))
363      return npm.get("version").getAsString();
364    else if (npm.get("name").getAsString().startsWith("hl7.fhir.r2.") || npm.get("name").getAsString().startsWith("hl7.fhir.r2b.") || npm.get("name").getAsString().startsWith("hl7.fhir.r3.") || npm.get("name").getAsString().startsWith("hl7.fhir.r4.") || npm.get("name").getAsString().startsWith("hl7.fhir.r5."))
365      return npm.get("version").getAsString();
366    else {        
367      JsonObject dep = npm.getAsJsonObject("dependencies");
368      if (dep == null)
369        throw new FHIRException("no dependencies found in the Package definition");
370      for (Entry<String, JsonElement> e : dep.entrySet()) {
371        if (Utilities.existsInList(e.getKey(), "hl7.fhir.r2.core", "hl7.fhir.r2b.core", "hl7.fhir.r3.core", "hl7.fhir.r4.core"))
372          return e.getValue().getAsString();
373        if (Utilities.existsInList(e.getKey(), "hl7.fhir.core")) // while all packages are updated
374          return e.getValue().getAsString();
375      }
376      if (npm.has("fhirVersions")) {
377        return npm.getAsJsonArray("fhirVersions").get(0).getAsString();
378      }
379      // legacy simplifier support:
380      if (dep.has("simplifier.core.r4"))
381        return "4.0";
382      if (dep.has("simplifier.core.r3"))
383        return "3.0";
384      if (dep.has("simplifier.core.r2"))
385        return "2.0";
386      throw new FHIRException("no core dependency or FHIR Version found in the Package definition");
387    }
388  }
389
390  public String summary() {
391    if (path != null)
392      return path;
393    else
394      return "memory";
395  }
396
397  public boolean isType(PackageType template) {
398    return template.getCode().equals(type());
399  }
400
401  public String type() {
402    return npm.get("type").getAsString();
403  }
404
405  public String description() {
406    return npm.has("description") ? npm.get("description").getAsString() : null;
407  }
408
409  public String getPath() {
410    return path;
411  }
412
413  public IniFile getCache() {
414    return cache;
415  }
416
417  /**
418   * only for use by the package manager itself
419   * 
420   * @param path
421   */
422  public void setPath(String path) {
423    this.path = path;
424  }
425
426  public String getWebLocation() {
427    if (npm.has("url"))
428      return npm.get("url").getAsString();
429    else
430      return npm.get("canonical").getAsString();
431  }
432
433  public boolean hasFile(String folder, String name) throws IOException {
434    String file = folder+"/"+name;
435    return content.containsKey(file);
436  }
437
438  public InputStream loadResource(String type, String id) throws IOException {
439    String file = type+"-"+id+".json";
440    if (content.containsKey("package/"+file))
441      return new ByteArrayInputStream(content.get("package/"+file));
442    else {
443      File f = new File(Utilities.path(path, "package", file));
444      if (f.exists())
445        return new FileInputStream(f);
446      else
447        return null;
448    }
449  }
450
451  /** special case when playing around inside the package **/
452  public Map<String, byte[]> getContent() {
453    return content;
454  }
455
456  public static NpmPackage fromFolder(String folder, String... exemptions) throws IOException {
457    return fromFolder(folder, null, exemptions);
458  }
459
460  public static NpmPackage fromFolder(String folder, PackageType defType, String... exemptions) throws IOException {
461    NpmPackage res = new NpmPackage(null);
462    loadFiles(res, folder, new File(folder), exemptions);
463    if (!res.content.containsKey("package/package.json") && defType != null)
464      res.content.put("package/package.json", TextFile.stringToBytes("{ \"type\" : \""+defType.getCode()+"\"}", false));
465    res.npm = (JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/package.json")));
466    return res;
467  }
468
469  private static void loadFiles(NpmPackage res, String base, File folder, String... exemptions) throws FileNotFoundException, IOException {
470    for (File f : folder.listFiles()) {
471      if (!f.getName().equals(".git") || !Utilities.existsInList(f.getName(), exemptions)) {
472        if (f.isDirectory()) 
473          loadFiles(res, base, f);
474        else {
475          String name = f.getAbsolutePath().substring(base.length()+1).replace("\\",  "/");
476          byte[] cnt = TextFile.fileToBytes(f);
477          res.content.put(name, cnt);
478        }
479      }
480    }
481  }
482
483  public void unPack(String dir) throws IOException {
484    for (String s : content.keySet()) {
485      String fn = Utilities.path(dir, s);
486      String dn = Utilities.getDirectoryForFile(fn);
487      Utilities.createDirectory(dn);
488      TextFile.bytesToFile(content.get(s), fn);
489    }
490    if (path != null)
491      FileUtils.copyDirectory(new File(path), new File(dir));      
492  }
493
494  public void save(OutputStream stream) throws IOException {
495    TarArchiveOutputStream tar;
496    ByteArrayOutputStream OutputStream;
497    BufferedOutputStream bufferedOutputStream;
498    GzipCompressorOutputStream gzipOutputStream;
499
500    OutputStream = new ByteArrayOutputStream();
501    bufferedOutputStream = new BufferedOutputStream(OutputStream);
502    gzipOutputStream = new GzipCompressorOutputStream(bufferedOutputStream);
503    tar = new TarArchiveOutputStream(gzipOutputStream);
504
505    NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder();
506    indexer.start();
507
508    for (String s : content.keySet()) {
509      byte[] b = content.get(s);
510      if (s.startsWith("package/")) { 
511        indexer.seeFile(tail(s), b);
512      }
513      if (!s.endsWith(".index.json") && !s.equals("package/package.json")) {
514        TarArchiveEntry entry = new TarArchiveEntry(s);
515        entry.setSize(b.length);
516        tar.putArchiveEntry(entry);
517        tar.write(b);
518        tar.closeArchiveEntry();
519      }
520    }
521    byte[] cnt = new GsonBuilder().setPrettyPrinting().create().toJson(npm).getBytes(Charset.forName("UTF-8"));
522    TarArchiveEntry entry = new TarArchiveEntry("package/package.json");
523    entry.setSize(cnt.length);
524    tar.putArchiveEntry(entry);
525    tar.write(cnt);
526    tar.closeArchiveEntry();
527
528    cnt = indexer.build().getBytes(Charset.forName("UTF-8"));
529    entry = new TarArchiveEntry("package/.index.json");
530    entry.setSize(cnt.length);
531    tar.putArchiveEntry(entry);
532    tar.write(cnt);
533    tar.closeArchiveEntry();
534
535    tar.finish();
536    tar.close();
537    gzipOutputStream.close();
538    bufferedOutputStream.close();
539    OutputStream.close();
540    byte[] b = OutputStream.toByteArray();
541    stream.write(b);
542  }
543
544  private String tail(String s) {
545    return s.substring(s.lastIndexOf("/")+1);
546  }
547  
548  public List<String> dependencies() {
549    List<String> res = new ArrayList<>();
550    if (npm.has("dependencies")) {
551      for (Entry<String, JsonElement> e : npm.getAsJsonObject("dependencies").entrySet()) {
552        res.add(e.getKey()+"#"+e.getValue().getAsString());
553      }
554    }
555    return res;
556  }
557}
558