001package org.hl7.fhir.utilities.cache; 002 003/*- 004 * #%L 005 * org.hl7.fhir.utilities 006 * %% 007 * Copyright (C) 2014 - 2019 Health Level 7 008 * %% 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 * #L% 021 */ 022 023 024import java.io.BufferedOutputStream; 025import java.io.ByteArrayInputStream; 026import java.io.ByteArrayOutputStream; 027import java.io.File; 028import java.io.FileInputStream; 029import java.io.FileNotFoundException; 030import java.io.IOException; 031import java.io.InputStream; 032import java.io.OutputStream; 033import java.nio.charset.Charset; 034import java.util.ArrayList; 035import java.util.Collections; 036import java.util.HashMap; 037import java.util.List; 038import java.util.Map; 039import java.util.Map.Entry; 040import java.util.zip.ZipEntry; 041import java.util.zip.ZipInputStream; 042 043import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 044import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 045import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; 046import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; 047import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; 048import org.apache.commons.io.FileUtils; 049import org.apache.commons.io.IOUtils; 050import org.hl7.fhir.exceptions.FHIRException; 051import org.hl7.fhir.utilities.IniFile; 052import org.hl7.fhir.utilities.TextFile; 053import org.hl7.fhir.utilities.Utilities; 054import org.hl7.fhir.utilities.cache.PackageCacheManager.PackageEntry; 055import org.hl7.fhir.utilities.cache.PackageGenerator.PackageType; 056import org.hl7.fhir.utilities.json.JsonTrackingParser; 057 058import com.google.common.base.Charsets; 059import com.google.gson.GsonBuilder; 060import com.google.gson.JsonArray; 061import com.google.gson.JsonElement; 062import com.google.gson.JsonObject; 063 064/** 065 * info and loader for a package 066 * 067 * Packages may exist on disk in the cache, or purely in memory when they are loaded on the fly 068 * 069 * Packages are contained in subfolders (see the package spec). The FHIR resources will be in "package" 070 * 071 * @author Grahame Grieve 072 * 073 */ 074public class NpmPackage { 075 076 private String path; 077 private List<String> folders = new ArrayList<String>(); 078 private Map<String, byte[]> content = new HashMap<String, byte[]>(); 079 private JsonObject npm; 080 private IniFile cache; 081 private Map<String, List<String>> types = new HashMap<>(); 082 private Map<String, String> urls = new HashMap<>(); 083 084 public NpmPackage(JsonObject npm, Map<String, byte[]> content, List<String> folders) { 085 this.path = null; 086 this.content = content; 087 this.npm = npm; 088 this.folders = folders; 089 } 090 091 public NpmPackage(String path) throws IOException { 092 this.path = path; 093 if (path != null) { 094 for (String f : sorted(new File(path).list())) { 095 if (new File(Utilities.path(path, f)).isDirectory()) { 096 folders.add(f); 097 } 098 } 099 npm = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(Utilities.path(path, "package", "package.json"))); 100 File ji = new File(Utilities.path(path, "package", ".index.json")); 101 if (ji.exists()) { 102 readIndexFile((JsonObject) new com.google.gson.JsonParser().parse(TextFile.fileToString(ji))); 103 } 104 cache = new IniFile(Utilities.path(path, "cache.ini")); 105 } 106 } 107 108 private void readIndexFile(JsonObject index) throws IOException { 109 for (JsonElement e : index.getAsJsonArray("files")) { 110 JsonObject file = (JsonObject) e; 111 String type = file.get("resourceType").getAsString(); 112 String name = file.get("filename").getAsString(); 113 String url = file.has("url") ? file.get("url").getAsString() : null; 114 String version = file.has("version") ? file.get("version").getAsString() : null; 115 if (!types.containsKey(type)) 116 types.put(type, new ArrayList<>()); 117 types.get(type).add(name); 118 if (version != null) 119 url = url + "|" + version; 120 urls.put(url, name); 121 } 122 } 123 124 private List<String> sorted(String[] keys) { 125 List<String> names = new ArrayList<String>(); 126 for (String s : keys) 127 names.add(s); 128 Collections.sort(names); 129 return names; 130 } 131 132 private static final int BUFFER_SIZE = 1024; 133 134 public static NpmPackage fromPackage(InputStream tgz) throws IOException { 135 return fromPackage(tgz, false); 136 } 137 138 public static NpmPackage fromPackage(InputStream tgz, boolean progress) throws IOException { 139 NpmPackage res = new NpmPackage(null); 140 boolean hasIndex = false; 141 GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(tgz); 142 try (TarArchiveInputStream tarIn = new TarArchiveInputStream(gzipIn)) { 143 TarArchiveEntry entry; 144 145 int i = 0; 146 int c = 12; 147 while ((entry = (TarArchiveEntry) tarIn.getNextEntry()) != null) { 148 i++; 149 if (entry.isDirectory()) { 150 res.folders.add(entry.getName()); 151 } else { 152 hasIndex = hasIndex || "package/.index.json".equals(entry.getName()); 153 int count; 154 byte data[] = new byte[BUFFER_SIZE]; 155 156 ByteArrayOutputStream fos = new ByteArrayOutputStream(); 157 try (BufferedOutputStream dest = new BufferedOutputStream(fos, BUFFER_SIZE)) { 158 while ((count = tarIn.read(data, 0, BUFFER_SIZE)) != -1) { 159 dest.write(data, 0, count); 160 } 161 } 162 fos.close(); 163 res.content.put(entry.getName(), fos.toByteArray()); 164 } 165 i++; 166 if (progress && i % 50 == 0) { 167 c++; 168 System.out.print("."); 169 if (c == 120) { 170 System.out.println(""); 171 System.out.print(" "); 172 c = 2; 173 } 174 } 175 } 176 } 177 if (!hasIndex) { 178 NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder(); 179 indexer.start(); 180 for (String n : res.getContent().keySet()) { 181 if (n.startsWith("package/")) { 182 indexer.seeFile(n.substring(8), res.content.get(n)); 183 } 184 } 185 res.content.put("package/.index.json", indexer.build().getBytes(Charsets.UTF_8)); 186 } 187 res.npm = JsonTrackingParser.parseJson(res.content.get("package/package.json")); 188 res.readIndexFile((JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/.index.json")))); 189 return res; 190 } 191 192 193 public static NpmPackage fromZip(InputStream stream, boolean dropRootFolder) throws IOException { 194 NpmPackage res = new NpmPackage(null); 195 ZipInputStream zip = new ZipInputStream(stream); 196 ZipEntry ze; 197 while ((ze = zip.getNextEntry()) != null) { 198 int size; 199 byte[] buffer = new byte[2048]; 200 201 ByteArrayOutputStream bytes = new ByteArrayOutputStream(); 202 BufferedOutputStream bos = new BufferedOutputStream(bytes, buffer.length); 203 204 while ((size = zip.read(buffer, 0, buffer.length)) != -1) { 205 bos.write(buffer, 0, size); 206 } 207 bos.flush(); 208 bos.close(); 209 if (bytes.size() > 0) 210 if (dropRootFolder) 211 res.content.put(ze.getName().substring(ze.getName().indexOf("/")+1), bytes.toByteArray()); 212 else 213 res.content.put(ze.getName(), bytes.toByteArray()); 214 215 zip.closeEntry(); 216 } 217 zip.close(); 218 res.npm = (JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/package.json"))); 219 if (res.content.containsKey("package/.index.json")) 220 res.readIndexFile((JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/.index.json")))); 221 return res; 222 } 223 224 225 /** 226 * Accessing the contents of the package - get a list of files in a subfolder of the package 227 * 228 * @param folder 229 * @return 230 * @throws IOException 231 */ 232 public List<String> list(String folder) throws IOException { 233 List<String> res = new ArrayList<String>(); 234 if (path != null) { 235 File f = new File(Utilities.path(path, folder)); 236 if (f.exists() && f.isDirectory()) 237 for (String s : f.list()) 238 res.add(s); 239 } else { 240 for (String s : content.keySet()) { 241 if (s.startsWith(folder+"/") && !s.substring(folder.length()+2).contains("/")) 242 res.add(s.substring(folder.length()+1)); 243 } 244 } 245 return res; 246 } 247 248 public List<String> listResources(String... types) throws IOException { 249 List<String> res = new ArrayList<String>(); 250 if (this.types.size() > 0) { 251 for (String s : types) { 252 if (this.types.containsKey(s)) 253 res.addAll(this.types.get(s)); 254 } 255 } else { 256 List<String> files = list("package"); 257 for (String s : files) { 258 String[] n = s.split("\\-"); 259 if (Utilities.existsInList(n[0], types)) 260 res.add(s); 261 } 262 } 263 return res; 264 } 265 266 /** 267 * Copies all the files in the package folder [folder] to the nominated dest, 268 * and returns a list of all the file names copied 269 * 270 * @param folder 271 * @return 272 * @throws IOException 273 */ 274 public List<String> copyTo(String folder, String dest) throws IOException { 275 List<String> res = new ArrayList<String>(); 276 if (path != null) { 277 copyToDest(Utilities.path(path, folder), Utilities.path(path, folder), dest, res); 278 } else { 279 for (Entry<String, byte[]> e : content.entrySet()) { 280 if (e.getKey().startsWith(folder+"/")) { 281 String s = e.getKey().substring(folder.length()+1); 282 res.add(s); 283 String dst = Utilities.path(dest, s); 284 String dstDir = Utilities.getDirectoryForFile(dst); 285 Utilities.createDirectory(dstDir); 286 TextFile.bytesToFile(e.getValue(), dst); 287 } 288 } 289 } 290 return res; 291 } 292 293 private void copyToDest(String base, String folder, String dest, List<String> res) throws IOException { 294 for (File f : new File(folder).listFiles()) { 295 if (f.isDirectory()) { 296 copyToDest(base, f.getAbsolutePath(), Utilities.path(dest, f.getName()), res); 297 } else { 298 String dst = Utilities.path(dest, f.getName()); 299 FileUtils.copyFile(f, new File(dst), true); 300 res.add(f.getAbsolutePath().substring(base.length()+1)); 301 } 302 } 303 } 304 305 /** 306 * get a stream that contains the contents of one of the files in a folder 307 * 308 * @param folder 309 * @param file 310 * @return 311 * @throws IOException 312 */ 313 public InputStream load(String folder, String file) throws IOException { 314 if (content.containsKey(folder+"/"+file)) 315 return new ByteArrayInputStream(content.get(folder+"/"+file)); 316 else { 317 File f = new File(Utilities.path(path, folder, file)); 318 if (f.exists()) 319 return new FileInputStream(f); 320 throw new IOException("Unable to find the file "+folder+"/"+file+" in the package "+name()); 321 } 322 } 323 324 /** 325 * Handle to the package json file 326 * 327 * @return 328 */ 329 public JsonObject getNpm() { 330 return npm; 331 } 332 333 /** 334 * convenience method for getting the package name 335 * @return 336 */ 337 public String name() { 338 return npm.get("name").getAsString(); 339 } 340 341 public String date() { 342 return npm.has("date") ? npm.get("date").getAsString() : null; 343 } 344 345 public String canonical() { 346 return npm.has("canonical") ? npm.get("canonical").getAsString() : null; 347 } 348 349 /** 350 * convenience method for getting the package version 351 * @return 352 */ 353 public String version() { 354 return npm.get("version").getAsString(); 355 } 356 357 /** 358 * convenience method for getting the package fhir version 359 * @return 360 */ 361 public String fhirVersion() { 362 if ("hl7.fhir.core".equals(npm.get("name").getAsString())) 363 return npm.get("version").getAsString(); 364 else if (npm.get("name").getAsString().startsWith("hl7.fhir.r2.") || npm.get("name").getAsString().startsWith("hl7.fhir.r2b.") || npm.get("name").getAsString().startsWith("hl7.fhir.r3.") || npm.get("name").getAsString().startsWith("hl7.fhir.r4.") || npm.get("name").getAsString().startsWith("hl7.fhir.r5.")) 365 return npm.get("version").getAsString(); 366 else { 367 JsonObject dep = npm.getAsJsonObject("dependencies"); 368 if (dep == null) 369 throw new FHIRException("no dependencies found in the Package definition"); 370 for (Entry<String, JsonElement> e : dep.entrySet()) { 371 if (Utilities.existsInList(e.getKey(), "hl7.fhir.r2.core", "hl7.fhir.r2b.core", "hl7.fhir.r3.core", "hl7.fhir.r4.core")) 372 return e.getValue().getAsString(); 373 if (Utilities.existsInList(e.getKey(), "hl7.fhir.core")) // while all packages are updated 374 return e.getValue().getAsString(); 375 } 376 if (npm.has("fhirVersions")) { 377 return npm.getAsJsonArray("fhirVersions").get(0).getAsString(); 378 } 379 // legacy simplifier support: 380 if (dep.has("simplifier.core.r4")) 381 return "4.0"; 382 if (dep.has("simplifier.core.r3")) 383 return "3.0"; 384 if (dep.has("simplifier.core.r2")) 385 return "2.0"; 386 throw new FHIRException("no core dependency or FHIR Version found in the Package definition"); 387 } 388 } 389 390 public String summary() { 391 if (path != null) 392 return path; 393 else 394 return "memory"; 395 } 396 397 public boolean isType(PackageType template) { 398 return template.getCode().equals(type()); 399 } 400 401 public String type() { 402 return npm.get("type").getAsString(); 403 } 404 405 public String description() { 406 return npm.has("description") ? npm.get("description").getAsString() : null; 407 } 408 409 public String getPath() { 410 return path; 411 } 412 413 public IniFile getCache() { 414 return cache; 415 } 416 417 /** 418 * only for use by the package manager itself 419 * 420 * @param path 421 */ 422 public void setPath(String path) { 423 this.path = path; 424 } 425 426 public String getWebLocation() { 427 if (npm.has("url")) 428 return npm.get("url").getAsString(); 429 else 430 return npm.get("canonical").getAsString(); 431 } 432 433 public boolean hasFile(String folder, String name) throws IOException { 434 String file = folder+"/"+name; 435 return content.containsKey(file); 436 } 437 438 public InputStream loadResource(String type, String id) throws IOException { 439 String file = type+"-"+id+".json"; 440 if (content.containsKey("package/"+file)) 441 return new ByteArrayInputStream(content.get("package/"+file)); 442 else { 443 File f = new File(Utilities.path(path, "package", file)); 444 if (f.exists()) 445 return new FileInputStream(f); 446 else 447 return null; 448 } 449 } 450 451 /** special case when playing around inside the package **/ 452 public Map<String, byte[]> getContent() { 453 return content; 454 } 455 456 public static NpmPackage fromFolder(String folder, String... exemptions) throws IOException { 457 return fromFolder(folder, null, exemptions); 458 } 459 460 public static NpmPackage fromFolder(String folder, PackageType defType, String... exemptions) throws IOException { 461 NpmPackage res = new NpmPackage(null); 462 loadFiles(res, folder, new File(folder), exemptions); 463 if (!res.content.containsKey("package/package.json") && defType != null) 464 res.content.put("package/package.json", TextFile.stringToBytes("{ \"type\" : \""+defType.getCode()+"\"}", false)); 465 res.npm = (JsonObject) new com.google.gson.JsonParser().parse(new String(res.content.get("package/package.json"))); 466 return res; 467 } 468 469 private static void loadFiles(NpmPackage res, String base, File folder, String... exemptions) throws FileNotFoundException, IOException { 470 for (File f : folder.listFiles()) { 471 if (!f.getName().equals(".git") || !Utilities.existsInList(f.getName(), exemptions)) { 472 if (f.isDirectory()) 473 loadFiles(res, base, f); 474 else { 475 String name = f.getAbsolutePath().substring(base.length()+1).replace("\\", "/"); 476 byte[] cnt = TextFile.fileToBytes(f); 477 res.content.put(name, cnt); 478 } 479 } 480 } 481 } 482 483 public void unPack(String dir) throws IOException { 484 for (String s : content.keySet()) { 485 String fn = Utilities.path(dir, s); 486 String dn = Utilities.getDirectoryForFile(fn); 487 Utilities.createDirectory(dn); 488 TextFile.bytesToFile(content.get(s), fn); 489 } 490 if (path != null) 491 FileUtils.copyDirectory(new File(path), new File(dir)); 492 } 493 494 public void save(OutputStream stream) throws IOException { 495 TarArchiveOutputStream tar; 496 ByteArrayOutputStream OutputStream; 497 BufferedOutputStream bufferedOutputStream; 498 GzipCompressorOutputStream gzipOutputStream; 499 500 OutputStream = new ByteArrayOutputStream(); 501 bufferedOutputStream = new BufferedOutputStream(OutputStream); 502 gzipOutputStream = new GzipCompressorOutputStream(bufferedOutputStream); 503 tar = new TarArchiveOutputStream(gzipOutputStream); 504 505 NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder(); 506 indexer.start(); 507 508 for (String s : content.keySet()) { 509 byte[] b = content.get(s); 510 if (s.startsWith("package/")) { 511 indexer.seeFile(tail(s), b); 512 } 513 if (!s.endsWith(".index.json") && !s.equals("package/package.json")) { 514 TarArchiveEntry entry = new TarArchiveEntry(s); 515 entry.setSize(b.length); 516 tar.putArchiveEntry(entry); 517 tar.write(b); 518 tar.closeArchiveEntry(); 519 } 520 } 521 byte[] cnt = new GsonBuilder().setPrettyPrinting().create().toJson(npm).getBytes(Charset.forName("UTF-8")); 522 TarArchiveEntry entry = new TarArchiveEntry("package/package.json"); 523 entry.setSize(cnt.length); 524 tar.putArchiveEntry(entry); 525 tar.write(cnt); 526 tar.closeArchiveEntry(); 527 528 cnt = indexer.build().getBytes(Charset.forName("UTF-8")); 529 entry = new TarArchiveEntry("package/.index.json"); 530 entry.setSize(cnt.length); 531 tar.putArchiveEntry(entry); 532 tar.write(cnt); 533 tar.closeArchiveEntry(); 534 535 tar.finish(); 536 tar.close(); 537 gzipOutputStream.close(); 538 bufferedOutputStream.close(); 539 OutputStream.close(); 540 byte[] b = OutputStream.toByteArray(); 541 stream.write(b); 542 } 543 544 private String tail(String s) { 545 return s.substring(s.lastIndexOf("/")+1); 546 } 547 548 public List<String> dependencies() { 549 List<String> res = new ArrayList<>(); 550 if (npm.has("dependencies")) { 551 for (Entry<String, JsonElement> e : npm.getAsJsonObject("dependencies").entrySet()) { 552 res.add(e.getKey()+"#"+e.getValue().getAsString()); 553 } 554 } 555 return res; 556 } 557} 558