001package org.hl7.fhir.utilities.cache; 002 003/*- 004 * #%L 005 * org.hl7.fhir.utilities 006 * %% 007 * Copyright (C) 2014 - 2019 Health Level 7 008 * %% 009 * Licensed under the Apache License, Version 2.0 (the "License"); 010 * you may not use this file except in compliance with the License. 011 * You may obtain a copy of the License at 012 * 013 * http://www.apache.org/licenses/LICENSE-2.0 014 * 015 * Unless required by applicable law or agreed to in writing, software 016 * distributed under the License is distributed on an "AS IS" BASIS, 017 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 018 * See the License for the specific language governing permissions and 019 * limitations under the License. 020 * #L% 021 */ 022 023 024import java.io.BufferedOutputStream; 025import java.io.ByteArrayOutputStream; 026import java.io.File; 027import java.io.FileInputStream; 028import java.io.IOException; 029import java.io.InputStream; 030import java.net.HttpURLConnection; 031import java.net.URL; 032import java.net.URLConnection; 033import java.sql.Timestamp; 034import java.time.Instant; 035import java.util.ArrayList; 036import java.util.Collections; 037import java.util.HashMap; 038import java.util.List; 039import java.util.Map; 040import java.util.Map.Entry; 041 042import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 043import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 044import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; 045import org.apache.commons.io.FileUtils; 046import org.hl7.fhir.exceptions.FHIRException; 047import org.hl7.fhir.utilities.IniFile; 048import org.hl7.fhir.utilities.TextFile; 049import org.hl7.fhir.utilities.Utilities; 050import org.hl7.fhir.utilities.json.JSONUtil; 051 052import com.google.gson.GsonBuilder; 053import com.google.gson.JsonArray; 054import com.google.gson.JsonElement; 055import com.google.gson.JsonObject; 056 057/** 058 * Package cache manager 059 * 060 * API: 061 * 062 * constructor 063 * getPackageUrl 064 * getPackageId 065 * findPackageCache 066 * addPackageToCache 067 * 068 * @author Grahame Grieve 069 * 070 */ 071public class PackageCacheManager { 072 073 public class VersionHistory { 074 private String id; 075 private String canonical; 076 private String current; 077 private Map<String, String> versions = new HashMap<>(); 078 public String getCanonical() { 079 return canonical; 080 } 081 public String getCurrent() { 082 return current; 083 } 084 public Map<String, String> getVersions() { 085 return versions; 086 } 087 public String getId() { 088 return id; 089 } 090 } 091 092 093 public class PackageEntry { 094 095 private byte[] bytes; 096 private String name; 097 098 public PackageEntry(String name) { 099 this.name = name; 100 } 101 102 public PackageEntry(String name, byte[] bytes) { 103 this.name = name; 104 this.bytes = bytes; 105 } 106 } 107 108 public static final String PACKAGE_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+$"; 109 public static final String PACKAGE_VERSION_REGEX = "^[a-z][a-z0-9\\_\\-]*(\\.[a-z0-9\\_\\-]+)+\\#[a-z0-9\\-\\_]+(\\.[a-z0-9\\-\\_]+)*$"; 110 111 private static final int BUFFER_SIZE = 1024; 112 private static final String CACHE_VERSION = "3"; // second version - see wiki page 113 private static final int ANALYSIS_VERSION = 2; 114 115 private String cacheFolder; 116 private boolean buildLoaded; 117 private JsonArray buildInfo; 118 private boolean progress = true; 119 private List<NpmPackage> temporaryPackages = new ArrayList<NpmPackage>(); 120 private Map<String, String> ciList = new HashMap<String, String>(); 121 private List<String> allUrls; 122 private Map<String, VersionHistory> historyCache = new HashMap<>(); 123 124 public PackageCacheManager(boolean userMode, int toolsVersion) throws IOException { 125 if (userMode) 126 cacheFolder = Utilities.path(System.getProperty("user.home"), ".fhir", "packages"); 127 else 128 cacheFolder = Utilities.path("var", "lib", ".fhir", "packages"); 129 if (!(new File(cacheFolder).exists())) 130 Utilities.createDirectory(cacheFolder); 131 if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists())) 132 TextFile.stringToFile("[cache]\r\nversion="+CACHE_VERSION+"\r\n\r\n[urls]\r\n\r\n[local]\r\n\r\n", Utilities.path(cacheFolder, "packages.ini"), false); 133 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 134 boolean save = false; 135 String v = ini.getStringProperty("cache", "version"); 136 if (!CACHE_VERSION.equals(v)) { 137 clearCache(); 138 ini.setStringProperty("cache", "version", CACHE_VERSION, null); 139 save = true; 140 } 141 save = initUrlMaps(ini, save); 142 if (save) { 143 if (!CACHE_VERSION.equals(ini.getStringProperty("cache", "version"))) { 144 throw new Error("what?"); 145 } 146 ini.save(); 147 } 148 } 149 150 public static String userDir() throws IOException { 151 return Utilities.path(System.getProperty("user.home"), ".fhir", "packages"); 152 } 153 154 // ========================= Utilities ============================================================================ 155 156 private List<String> sorted(String[] keys) { 157 List<String> names = new ArrayList<String>(); 158 for (String s : keys) 159 names.add(s); 160 Collections.sort(names); 161 return names; 162 } 163 164 private NpmPackage loadPackageInfo(String path) throws IOException { 165 NpmPackage pi = new NpmPackage(path); 166 return pi; 167 } 168 169 public String getFolder() { 170 return cacheFolder; 171 } 172 173 private void analysePackage(String dir) throws IOException { 174 NpmPackageIndexBuilder indexer = new NpmPackageIndexBuilder(); 175 indexer.start(); 176 int i = 0; 177 int c = 0; 178 File[] packages = new File(Utilities.path(dir, "package")).listFiles(); 179 for (File f : packages) { 180 indexer.seeFile(f.getName(), TextFile.fileToBytes(f)); 181 i++; 182 if (progress && i % 50 == 0) { 183 c++; 184 System.out.print("."); 185 if (c == 120) { 186 System.out.println(""); 187 System.out.print(" "); 188 c = 2; 189 } 190 } 191 } 192 TextFile.stringToFile(indexer.build(), Utilities.path(dir, "package", ".index.json")); 193 } 194 195 196 197 private NpmPackage checkCurrency(String id, NpmPackage p) { 198 // special case: current versions roll over, and we have to check their currency 199 try { 200 String url = ciList.get(id); 201 JsonObject json = fetchJson(Utilities.pathURL(url, "package.manifest.json")); 202 String currDate = JSONUtil.str(json, "date"); 203 String packDate = p.date(); 204 if (!currDate.equals(packDate)) 205 return null; // nup, we need a new copy 206 return p; 207 } catch (Exception e) { 208 return p; 209 } 210 } 211 212 private JsonObject fetchJson(String source) throws IOException { 213 URL url = new URL(source); 214 URLConnection c = url.openConnection(); 215 return (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(c.getInputStream())); 216 } 217 218 private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException { 219 try { 220 URL url = new URL(source); 221 URLConnection c = url.openConnection(); 222 return c.getInputStream(); 223 } catch (Exception e) { 224 if (optional) 225 return null; 226 else 227 throw new FHIRException(e.getMessage(), e); 228 } 229 } 230 231 // ========================= Full Cache Management ============================================================================ 232 233 private void clearCache() throws IOException { 234 for (File f : new File(cacheFolder).listFiles()) { 235 if (f.isDirectory()) 236 FileUtils.deleteDirectory(f); 237 else if (!f.getName().equals("packages.ini")) 238 FileUtils.forceDelete(f); 239 } 240 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 241 ini.removeSection("packages"); 242 ini.save(); 243 } 244 245// private void checkDeleteVersion(String id, String ver, int minVer) { 246// if (hasPackage(id, ver)) { 247// boolean del = true; 248// NpmPackage pck; 249// try { 250// pck = loadPackageFromCacheOnly(id, ver); 251// if (pck.getNpm().has("tools-version")) { 252// del = pck.getNpm().get("tools-version").getAsInt() < minVer; 253// } 254// } catch (Exception e) { 255// } 256// if (del) 257// try { 258// removePackage(id, ver); 259// } catch (IOException e) { 260// } 261// } 262// } 263 264//private void convertPackageCacheFrom1To2() throws IOException { 265//for (File f : new File(cacheFolder).listFiles()) { 266// if (f.isDirectory() && f.getName().contains("-")) { 267// String s = f.getName(); 268// int i = s.lastIndexOf("-"); 269// s = s.substring(0, i)+"#"+s.substring(i+1); 270// File nf = new File(Utilities.path(cacheFolder, s)); 271// if (!f.renameTo(nf)) 272// throw new IOException("Unable to rename "+f.getAbsolutePath()+" to "+nf.getAbsolutePath()); 273// } 274//} 275//} 276 277 // ========================= URL maps (while waiting for package registry) ============================================================================ 278 279 public boolean initUrlMaps(IniFile ini, boolean save) { 280 save = checkIniHasMapping("hl7.fhir.core", "http://hl7.org/fhir", ini) || save; 281 282 save = checkIniHasMapping("hl7.fhir.r2.core", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.core.tgz", ini) || save; 283 save = checkIniHasMapping("hl7.fhir.r2.examples", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.examples.tgz", ini) || save; 284 save = checkIniHasMapping("hl7.fhir.r2.elements", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.elements.tgz", ini) || save; 285 save = checkIniHasMapping("hl7.fhir.r2.expansions", "http://hl7.org/fhir/DSTU2/hl7.fhir.r2.expansions.tgz", ini) || save; 286 save = checkIniHasMapping("hl7.fhir.r2b.core", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.core.tgz", ini) || save; 287 save = checkIniHasMapping("hl7.fhir.r2b.examples", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.examples.tgz", ini) || save; 288 save = checkIniHasMapping("hl7.fhir.r2b.elements", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.elements.tgz", ini) || save; 289 save = checkIniHasMapping("hl7.fhir.r2b.expansions", "http://hl7.org/fhir/2016May/hl7.fhir.r2b.expansions.tgz", ini) || save; 290 save = checkIniHasMapping("hl7.fhir.r3.core", "http://hl7.org/fhir/STU3/hl7.fhir.r3.core.tgz", ini) || save; 291 save = checkIniHasMapping("hl7.fhir.r3.examples", "http://hl7.org/fhir/STU3/hl7.fhir.r3.examples.tgz", ini) || save; 292 save = checkIniHasMapping("hl7.fhir.r3.elements", "http://hl7.org/fhir/STU3/hl7.fhir.r3.elements.tgz", ini) || save; 293 save = checkIniHasMapping("hl7.fhir.r3.expansions", "http://hl7.org/fhir/STU3/hl7.fhir.r3.expansions.tgz", ini) || save; 294 save = checkIniHasMapping("hl7.fhir.r4.core", "http://hl7.org/fhir/R4/hl7.fhir.r4.core.tgz", ini) || save; 295 save = checkIniHasMapping("hl7.fhir.r4.examples", "http://hl7.org/fhir/R4/hl7.fhir.r4.examples.tgz", ini) || save; 296 save = checkIniHasMapping("hl7.fhir.r4.elements", "http://hl7.org/fhir/R4/hl7.fhir.r4.elements.tgz", ini) || save; 297 save = checkIniHasMapping("hl7.fhir.r4.expansions", "http://hl7.org/fhir/R4/hl7.fhir.r4.expansions.tgz", ini) || save; 298 299 save = checkIniHasMapping("hl7.fhir.r5.core", "http://build.fhir.org/package.tgz", ini) || save; 300 301 save = checkIniHasMapping("fhir.argonaut.ehr", "http://fhir.org/guides/argonaut", ini) || save; 302 save = checkIniHasMapping("fhir.argonaut.pd", "http://fhir.org/guides/argonaut-pd", ini) || save; 303 save = checkIniHasMapping("fhir.argonaut.scheduling", "http://fhir.org/guides/argonaut-scheduling", ini) || save; 304 save = checkIniHasMapping("fhir.hspc.acog", "http://hl7.org/fhir/fpar", ini) || save; 305 save = checkIniHasMapping("hl7.fhir.au.argonaut", "http://hl7.org.au/fhir/argonaut", ini) || save; 306 save = checkIniHasMapping("hl7.fhir.au.base", "http://hl7.org.au/fhir/base", ini) || save; 307 save = checkIniHasMapping("hl7.fhir.au.pd", "http://hl7.org.au/fhir/pd", ini) || save; 308 save = checkIniHasMapping("hl7.fhir.smart", "http://hl7.org/fhir/smart-app-launch", ini) || save; 309 save = checkIniHasMapping("hl7.fhir.snomed", "http://hl7.org/fhir/ig/snomed", ini) || save; 310 save = checkIniHasMapping("hl7.fhir.test.v10", "http://hl7.org/fhir/test-ig-10", ini) || save; 311 save = checkIniHasMapping("hl7.fhir.test.v30", "http://hl7.org/fhir/test", ini) || save; 312 save = checkIniHasMapping("hl7.fhir.us.breastcancer", "http://hl7.org/fhir/us/breastcancer", ini) || save; 313 save = checkIniHasMapping("hl7.fhir.us.ccda", "http://hl7.org/fhir/us/ccda", ini) || save; 314 save = checkIniHasMapping("hl7.fhir.us.cds.opioids", "http://hl7.org/fhir/ig/opioid-cds", ini) || save; 315 save = checkIniHasMapping("hl7.fhir.us.core", "http://hl7.org/fhir/us/core", ini) || save; 316 save = checkIniHasMapping("hl7.fhir.us.dafresearch", "http://hl7.org/fhir/us/daf-research", ini) || save; 317 save = checkIniHasMapping("hl7.fhir.us.ecr", "http://fhir.hl7.org/us/ecr", ini) || save; 318 save = checkIniHasMapping("hl7.fhir.us.hai", "http://hl7.org/fhir/us/hai", ini) || save; 319 save = checkIniHasMapping("hl7.fhir.us.hedis", "http://hl7.org/fhir/us/hedis", ini) || save; 320 save = checkIniHasMapping("hl7.fhir.us.meds", "http://hl7.org/fhir/us/meds", ini) || save; 321 save = checkIniHasMapping("hl7.fhir.us.qicore", "http://hl7.org/fhir/us/qicore", ini) || save; 322 save = checkIniHasMapping("hl7.fhir.us.sdc", "http://hl7.org/fhir/us/sdc", ini) || save; 323 save = checkIniHasMapping("hl7.fhir.us.sdcde", "http://hl7.org/fhir/us/sdcde", ini) || save; 324 save = checkIniHasMapping("hl7.fhir.uv.genomicsreporting", "http://hl7.org/fhir/uv/genomics-reporting", ini) || save; 325 save = checkIniHasMapping("hl7.fhir.uv.immds", "http://hl7.org/fhir/uv/cdsi", ini) || save; 326 save = checkIniHasMapping("hl7.fhir.uv.ips", "http://hl7.org/fhir/uv/ips", ini) || save; 327 save = checkIniHasMapping("hl7.fhir.uv.phd", "http://hl7.org/fhir/devices", ini) || save; 328 save = checkIniHasMapping("hl7.fhir.uv.vhdir", "http://hl7.org/fhir/ig/vhdir", ini) || save; 329 save = checkIniHasMapping("hl7.fhir.vn.base", "http://hl7.org/fhir/ig/vietnam", ini) || save; 330 save = checkIniHasMapping("hl7.fhir.vocabpoc", "http://hl7.org/fhir/ig/vocab-poc", ini) || save; 331 return save; 332 } 333 334 private boolean checkIniHasMapping(String pid, String curl, IniFile ini) { 335 if (curl.equals(ini.getStringProperty("urls", pid))) 336 return false; 337 ini.setStringProperty("urls", pid, curl, null); 338 return true; 339 } 340 341 public void recordMap(String url, String id) throws IOException { 342 if (url == null) 343 return; 344 345 if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists())) 346 throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #1"); 347 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 348 ini.setStringProperty("urls", id, url, null); 349 if (!CACHE_VERSION.equals(ini.getStringProperty("cache", "version"))) { 350 throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" cache version mismatch: expected '"+CACHE_VERSION+"', found '"+ini.getStringProperty("cache", "version")+"'"); 351 } 352 if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists())) 353 throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #2"); 354 355 ini.save(); 356 if (!(new File(Utilities.path(cacheFolder, "packages.ini")).exists())) 357 throw new Error("File "+Utilities.path(cacheFolder, "packages.ini")+" not found #3"); 358 } 359 360 public String getPackageUrl(String id) throws IOException { 361 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 362 return ini.getStringProperty("urls", id); 363 } 364 365 public String getPackageId(String url) throws IOException { 366 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 367 String[] ids = ini.getPropertyNames("urls"); 368 if (ids != null) { 369 for (String id : ids) { 370 if (url.equals(ini.getStringProperty("urls", id))) 371 return id; 372 } 373 } 374 return null; 375 } 376 377 public JsonArray loadFromBuildServer() throws IOException { 378 buildLoaded = true; // whether it succeeds or not 379 URL url = new URL("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis()); 380 HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 381 connection.setRequestMethod("GET"); 382 InputStream json = connection.getInputStream(); 383 buildInfo = (JsonArray) new com.google.gson.JsonParser().parse(TextFile.streamToString(json)); 384 385 for (JsonElement n : buildInfo) { 386 JsonObject o = (JsonObject) n; 387 if (o.has("url") && o.has("package-id") && o.get("package-id").getAsString().contains(".")) { 388 String u = o.get("url").getAsString(); 389 if (u.contains("/ImplementationGuide/")) 390 u = u.substring(0, u.indexOf("/ImplementationGuide/")); 391 recordMap(u, o.get("package-id").getAsString()); 392 ciList.put(o.get("package-id").getAsString(), "https://build.fhir.org/ig/"+o.get("repo").getAsString()); 393 } 394 } 395 return buildInfo; 396 } 397 398 public boolean isBuildLoaded() { 399 return buildLoaded; 400 } 401 402 403 public String buildPath(String url) { 404 for (JsonElement e : buildInfo) { 405 JsonObject j = (JsonObject) e; 406 if (j.has("url") && (url.equals(j.get("url").getAsString()) || j.get("url").getAsString().startsWith(url+"/ImplementationGuide"))) { 407 return "https://build.fhir.org/ig/"+j.get("repo").getAsString(); 408 } 409 } 410 return null; 411 } 412 413 public boolean checkBuildLoaded() throws IOException { 414 if (isBuildLoaded()) 415 return true; 416 loadFromBuildServer(); 417 return false; 418 } 419 420 public Map<String, String> getCiList() { 421 return ciList; 422 } 423 424 public List<String> getUrls() throws IOException { 425 if (allUrls == null) 426 { 427 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 428 allUrls = new ArrayList<>(); 429 for (String s : ini.getPropertyNames("urls")) 430 allUrls.add(ini.getStringProperty("urls", s)); 431 try { 432 URL url = new URL("https://raw.githubusercontent.com/FHIR/ig-registry/master/fhir-ig-list.json?nocache=" + System.currentTimeMillis()); 433 HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 434 connection.setRequestMethod("GET"); 435 InputStream json = connection.getInputStream(); 436 JsonObject packages = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json)); 437 JsonArray guides = packages.getAsJsonArray("guides"); 438 for (JsonElement g : guides) { 439 JsonObject gi = (JsonObject) g; 440 if (gi.has("canonical")) 441 if (!allUrls.contains(gi.get("canonical").getAsString())) 442 allUrls.add(gi.get("canonical").getAsString()); 443 } 444 } catch (Exception e) { 445 System.out.println("Listing known Implementation Guides failed: "+e.getMessage()); 446 } 447 } 448 return allUrls; 449 } 450 451 // ========================= Package API ============================================================================ 452 453 public void removePackage(String id, String ver) throws IOException { 454 String f = Utilities.path(cacheFolder, id+"#"+ver); 455 Utilities.clearDirectory(f); 456 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 457 ini.removeProperty("packages", id+"#"+ver); 458 ini.save(); 459 new File(f).delete(); 460 } 461 462 /** 463 * get the latest version of the package from what is in the cache 464 * @param id 465 * @return 466 * @throws IOException 467 */ 468 public NpmPackage loadPackageFromCacheOnly(String id) throws IOException { 469 List<String> l = sorted(new File(cacheFolder).list()); 470 for (int i = l.size()-1; i >= 0; i--) { 471 String f = l.get(i); 472 if (f.startsWith(id+"#")) { 473 return loadPackageInfo(Utilities.path(cacheFolder, f)); 474 } 475 } 476 return null; 477 } 478 479 /** 480 * Load the identified package from the cache - it it exists 481 * 482 * This is for special purpose only. Generally, use the loadPackage method 483 * 484 * @param id 485 * @param version 486 * @return 487 * @throws IOException 488 */ 489 public NpmPackage loadPackageFromCacheOnly(String id, String version) throws IOException { 490 for (NpmPackage p : temporaryPackages) { 491 if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) 492 return p; 493 } 494 for (String f : sorted(new File(cacheFolder).list())) { 495 if (f.equals(id+"#"+version)) { 496 return loadPackageInfo(Utilities.path(cacheFolder, f)); 497 } 498 } 499 if ("dev".equals(version)) 500 return loadPackageFromCacheOnly(id, "current"); 501 else 502 return null; 503 } 504 505 /** 506 * Add an already fetched package to the cache 507 */ 508 public NpmPackage addPackageToCache(String id, String version, InputStream tgz) throws IOException { 509 if (progress ) { 510 System.out.println("Installing "+id+"#"+(version == null ? "?" : version)+" to the package cache"); 511 System.out.print(" Fetching:"); 512 } 513 514 NpmPackage npm = NpmPackage.fromPackage(tgz, true); 515 516 recordMap(npm.canonical(), npm.name()); 517 518 if (progress ) 519 System.out.print("|"); 520 if (npm.name() == null || id == null || !id.equals(npm.name())) { 521 if (!id.equals("hl7.fhir.r5.core")) {// temporary work around 522 throw new IOException("Attempt to import a mis-identified package. Expected "+id+", got "+npm.name()); 523 } 524 } 525 if (version == null) 526 version = npm.version(); 527 528 String packRoot = Utilities.path(cacheFolder, id+"#"+version); 529 Utilities.createDirectory(packRoot); 530 Utilities.clearDirectory(packRoot); 531 532 int i = 0; 533 int c = 0; 534 int size = 0; 535 for (Entry<String, byte[]> e : npm.getContent().entrySet()) { 536 String fn = Utilities.path(packRoot, e.getKey()); 537 String dir = Utilities.getDirectoryForFile(fn); 538 if (!(new File(dir).exists())) 539 Utilities.createDirectory(dir); 540 TextFile.bytesToFile(e.getValue(), fn); 541 size = size + e.getValue().length; 542 i++; 543 if (progress && i % 50 == 0) { 544 c++; 545 System.out.print("."); 546 if (c == 120) { 547 System.out.println(""); 548 System.out.print(" "); 549 c = 2; 550 } 551 } 552 } 553 if (progress) { 554 System.out.println(""); 555 System.out.print(" Analysing"); 556 } 557 558 analysePackage(packRoot); // do this whether there's an index.json or not. 559 560 IniFile ini = new IniFile(Utilities.path(cacheFolder, "packages.ini")); 561 ini.setTimeStampFormat("yyyyMMddhhmmss"); 562 ini.setTimestampProperty("packages", id+"#"+version, Timestamp.from(Instant.now()), null); 563 ini.setIntegerProperty("package-sizes", id+"#"+version, size, null); 564 ini.save(); 565 if (progress) 566 System.out.println(" done."); 567 568 //todo: load dependencies 569 NpmPackage pck = loadPackageInfo(packRoot); 570 if (!id.equals(JSONUtil.str(npm.getNpm(), "name")) || !version.equals(JSONUtil.str(npm.getNpm(), "version"))) { 571 if (!id.equals(JSONUtil.str(npm.getNpm(), "name"))) { 572 npm.getNpm().addProperty("original-name", JSONUtil.str(npm.getNpm(), "name")); 573 npm.getNpm().remove("name"); 574 npm.getNpm().addProperty("name", id); 575 } 576 if (!version.equals(JSONUtil.str(npm.getNpm(), "version"))) { 577 npm.getNpm().addProperty("original-version", JSONUtil.str(npm.getNpm(), "version")); 578 npm.getNpm().remove("version"); 579 npm.getNpm().addProperty("version", version); 580 } 581 TextFile.stringToFile(new GsonBuilder().setPrettyPrinting().create().toJson(npm.getNpm()), Utilities.path(cacheFolder, id+"#"+version, "package", "package.json"), false); 582 } 583 584 return pck; 585 } 586 587 public NpmPackage loadPackage(String id) throws FHIRException, IOException { 588 throw new Error("Not done yet"); 589 } 590 591 public NpmPackage loadPackage(String id, String v) throws FHIRException, IOException { 592 NpmPackage p = loadPackageFromCacheOnly(id, v); 593 if (p != null) { 594 if ("current".equals(v)) { 595 p = checkCurrency(id, p); 596 } 597 if (p != null) 598 return p; 599 } 600 601 if ("dev".equals(v)) { 602 p = loadPackageFromCacheOnly(id, "current"); 603 p = checkCurrency(id, p); 604 if (p != null) 605 return p; 606 v = "current"; 607 } 608 609 String url = getPackageUrl(id); 610 if (url == null) 611 throw new FHIRException("Unable to resolve the package '"+id+"'"); 612 if (url.contains(".tgz")) { 613 InputStream stream = fetchFromUrlSpecific(url, true); 614 if (stream != null) 615 return addPackageToCache(id, v, stream); 616 throw new FHIRException("Unable to find the package source for '"+id+"' at "+url); 617 } 618 if (v == null) { 619 InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(url, "package.tgz"), true); 620 if (stream == null && isBuildLoaded()) { 621 stream = fetchFromUrlSpecific(Utilities.pathURL(buildPath(url), "package.tgz"), true); 622 } 623 if (stream != null) 624 return addPackageToCache(id, null, stream); 625 throw new FHIRException("Unable to find the package source for '"+id+"' at "+url); 626 } else if ("current".equals(v) && ciList.containsKey(id)){ 627 InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), true); 628 return addPackageToCache(id, v, stream); 629 } else { 630 String pu = Utilities.pathURL(url, "package-list.json"); 631 JsonObject json; 632 try { 633 json = fetchJson(pu); 634 } catch (Exception e) { 635 throw new FHIRException("Error fetching package list for "+id+" from "+pu+": "+e.getMessage(), e); 636 } 637 if (!id.equals(JSONUtil.str(json, "package-id"))) 638 throw new FHIRException("Package ids do not match in "+pu+": "+id+" vs "+JSONUtil.str(json, "package-id")); 639 for (JsonElement e : json.getAsJsonArray("list")) { 640 JsonObject vo = (JsonObject) e; 641 if (v.equals(JSONUtil.str(vo, "version"))) { 642 InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz"), true); 643 if (stream == null) 644 throw new FHIRException("Unable to find the package source for '"+id+"#"+v+"' at "+Utilities.pathURL(JSONUtil.str(vo, "path"), "package.tgz")); 645 return addPackageToCache(id, v, stream); 646 } 647 } 648// // special case: current version 649// if (id.equals("hl7.fhir.core") && v.equals(currentVersion)) { 650// InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("http://build.fhir.org", "package.tgz"), true); 651// if (stream == null) 652// throw new FHIRException("Unable to find the package source for '"+id+"#"+v+"' at "+Utilities.pathURL("http://build.fhir.org", "package.tgz")); 653// return addPackageToCache(id, v, stream); 654// } 655 throw new FHIRException("Unable to resolve version "+v+" for package "+id); 656 } 657 } 658 659 660// public void loadFromFolder(String packagesFolder) throws IOException { 661// for (File f : new File(packagesFolder).listFiles()) { 662// if (f.getName().endsWith(".tgz")) { 663// temporaryPackages.add(extractLocally(new FileInputStream(f), f.getName())); 664// } 665// } 666// } 667// 668 669 /** 670 * turn true if a package exists 671 * @param id 672 * @param version 673 * @return 674 */ 675 public boolean hasPackage(String id, String version) { 676 for (NpmPackage p : temporaryPackages) { 677 if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) 678 return true; 679 } 680 for (String f : sorted(new File(cacheFolder).list())) { 681 if (f.equals(id+"#"+version)) { 682 return true; 683 } 684 } 685 if ("dev".equals(version)) 686 return hasPackage(id, "current"); 687 else 688 return false; 689 } 690 691 692 /** 693 * List which versions of a package are available 694 * 695 * @param url 696 * @return 697 * @throws IOException 698 */ 699 700 public VersionHistory listVersions(String url) throws IOException { 701 if (historyCache.containsKey(url)) 702 return historyCache.get(url); 703 704 URL url1 = new URL(Utilities.pathURL(url, "package-list.json")+"?nocache=" + System.currentTimeMillis()); 705 HttpURLConnection connection = (HttpURLConnection) url1.openConnection(); 706 connection.setRequestMethod("GET"); 707 InputStream json = connection.getInputStream(); 708 JsonObject packageList = (JsonObject) new com.google.gson.JsonParser().parse(TextFile.streamToString(json)); 709 VersionHistory res = new VersionHistory(); 710 res.id = JSONUtil.str(packageList, "package-id"); 711 res.canonical = JSONUtil.str(packageList, "canonical"); 712 for (JsonElement j : packageList.getAsJsonArray("list")) { 713 JsonObject jo = (JsonObject) j; 714 if ("current".equals(JSONUtil.str(jo, "version"))) 715 res.current = JSONUtil.str(jo, "path"); 716 else 717 res.versions.put(JSONUtil.str(jo, "version"), JSONUtil.str(jo, "path")); 718 } 719 historyCache.put(url, res); 720 return res; 721 } 722 723 724 /** 725 * Clear the cache 726 * 727 * @throws IOException 728 */ 729 public void clear() throws IOException { 730 clearCache(); 731 } 732 733 public void loadFromFolder(String packagesFolder) throws IOException { 734 for (File f : new File(packagesFolder).listFiles()) { 735 if (f.getName().endsWith(".tgz")) { 736 temporaryPackages.add(NpmPackage.fromPackage(new FileInputStream(f))); 737 } 738 } 739 } 740}