001package org.hl7.fhir.utilities.npm; 002 003import java.io.*; 004 005import java.text.ParseException; 006import java.text.SimpleDateFormat; 007 008import java.util.*; 009 010import javax.annotation.Nonnull; 011import javax.annotation.Nullable; 012 013import lombok.Getter; 014import lombok.Setter; 015import lombok.With; 016import org.apache.commons.io.FileUtils; 017import org.hl7.fhir.exceptions.FHIRException; 018import org.hl7.fhir.utilities.IniFile; 019import org.hl7.fhir.utilities.TextFile; 020import org.hl7.fhir.utilities.Utilities; 021import org.hl7.fhir.utilities.VersionUtilities; 022import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 023import org.hl7.fhir.utilities.http.HTTPResult; 024import org.hl7.fhir.utilities.http.ManagedWebAccess; 025import org.hl7.fhir.utilities.json.model.JsonArray; 026import org.hl7.fhir.utilities.json.model.JsonElement; 027import org.hl7.fhir.utilities.json.model.JsonObject; 028import org.hl7.fhir.utilities.json.parser.JsonParser; 029import org.hl7.fhir.utilities.npm.PackageList.PackageListEntry; 030import org.hl7.fhir.utilities.settings.FhirSettings; 031import org.slf4j.Logger; 032import org.slf4j.LoggerFactory; 033 034/* 035 Copyright (c) 2011+, HL7, Inc. 036 All rights reserved. 037 038 Redistribution and use in source and binary forms, with or without modification, 039 are permitted provided that the following conditions are met: 040 041 * Redistributions of source code must retain the above copyright notice, this 042 list of conditions and the following disclaimer. 043 * Redistributions in binary form must reproduce the above copyright notice, 044 this list of conditions and the following disclaimer in the documentation 045 and/or other materials provided with the distribution. 046 * Neither the name of HL7 nor the names of its contributors may be used to 047 endorse or promote products derived from this software without specific 048 prior written permission. 049 050 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 051 ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 052 WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 053 IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 054 INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 055 NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 056 PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 057 WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 058 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 059 POSSIBILITY OF SUCH DAMAGE. 060 061 */ 062 063/** 064 * This is a package cache manager implementation that uses a local disk cache 065 * 066 * <p> 067 * API: 068 * <p> 069 * constructor 070 * getPackageUrl 071 * getPackageId 072 * findPackageCache 073 * addPackageToCache 074 * 075 * @author Grahame Grieve 076 */ 077public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager { 078 079 private final FilesystemPackageCacheManagerLocks locks; 080 081 private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters; 082 083 // When running in testing mode, some packages are provided from the test case repository rather than by the normal means 084 // the PackageProvider is responsible for this. if no package provider is defined, or it declines to handle the package, 085 // then the normal means will be used 086 public interface IPackageProvider { 087 boolean handlesPackage(String id, String version); 088 InputStreamWithSrc provide(String id, String version) throws IOException; 089 } 090 091 private static IPackageProvider packageProvider; 092 public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$"; 093 public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$"; 094 public static final String PACKAGE_VERSION_REGEX_OPT = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+(\\#[A-Za-z0-9\\-\\_]+(\\.[A-Za-z0-9\\-\\_]+)*)?$"; 095 private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class); 096 private static final String CACHE_VERSION = "3"; // second version - see wiki page 097 098 @Nonnull 099 private final File cacheFolder; 100 101 private final List<NpmPackage> temporaryPackages = new ArrayList<>(); 102 private boolean buildLoaded = false; 103 private final Map<String, String> ciList = new HashMap<>(); 104 private JsonArray buildInfo; 105 private boolean suppressErrors; 106 107 @Setter 108 @Getter 109 private boolean minimalMemory; 110 111 public static class Builder { 112 113 @Getter 114 private final File cacheFolder; 115 116 @With 117 @Getter 118 private final List<PackageServer> packageServers; 119 120 @With 121 @Getter 122 private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters; 123 124 public Builder() throws IOException { 125 this.cacheFolder = getUserCacheFolder(); 126 this.packageServers = getPackageServersFromFHIRSettings(); 127 this.lockParameters = null; 128 } 129 130 private Builder(File cacheFolder, List<PackageServer> packageServers, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) { 131 this.cacheFolder = cacheFolder; 132 this.packageServers = packageServers; 133 this.lockParameters = lockParameters; 134 } 135 136 private File getUserCacheFolder() throws IOException { 137 return ManagedFileAccess.file(Utilities.path(System.getProperty("user.home"), ".fhir", "packages")); 138 } 139 140 private List<PackageServer> getPackageServersFromFHIRSettings() { 141 List<PackageServer> packageServers = new ArrayList<>(getConfiguredServers()); 142 if (!isIgnoreDefaultPackageServers()) { 143 packageServers.addAll(getDefaultServers()); 144 } 145 return packageServers; 146 } 147 148 protected boolean isIgnoreDefaultPackageServers() { 149 return FhirSettings.isIgnoreDefaultPackageServers(); 150 } 151 152 @Nonnull 153 protected List<PackageServer> getDefaultServers() { 154 return PackageServer.defaultServers(); 155 } 156 157 protected List<PackageServer> getConfiguredServers() { 158 return PackageServer.getConfiguredServers(); 159 } 160 161 public Builder withCacheFolder(String cacheFolderPath) throws IOException { 162 File cacheFolder = ManagedFileAccess.file(cacheFolderPath); 163 if (!cacheFolder.exists()) { 164 throw new FHIRException("The folder '" + cacheFolder + "' could not be found"); 165 } 166 return new Builder(cacheFolder, this.packageServers, this.lockParameters); 167 } 168 169 public Builder withSystemCacheFolder() throws IOException { 170 final File systemCacheFolder; 171 if (Utilities.isWindows()) { 172 systemCacheFolder = ManagedFileAccess.file(Utilities.path(System.getenv("ProgramData"), ".fhir", "packages")); 173 } else { 174 systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages")); 175 } 176 return new Builder(systemCacheFolder, this.packageServers, this.lockParameters); 177 } 178 179 public Builder withTestingCacheFolder() throws IOException { 180 return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.lockParameters); 181 } 182 183 public FilesystemPackageCacheManager build() throws IOException { 184 final FilesystemPackageCacheManagerLocks locks; 185 try { 186 locks = FilesystemPackageCacheManagerLocks.getFilesystemPackageCacheManagerLocks(cacheFolder); 187 } catch (RuntimeException e) { 188 if (e.getCause() instanceof IOException) { 189 throw (IOException) e.getCause(); 190 } else { 191 throw e; 192 } 193 } 194 return new FilesystemPackageCacheManager(cacheFolder, packageServers, locks, lockParameters); 195 } 196 } 197 198 private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List<PackageServer> packageServers, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException { 199 super(packageServers); 200 this.cacheFolder = cacheFolder; 201 this.locks = locks; 202 this.lockParameters = lockParameters; 203 prepareCacheFolder(); 204 } 205 206 /** 207 * Check if the cache folder exists and is valid. 208 * <p> 209 * If it doesn't exist, create it. 210 * <p> 211 * If it does exist and isn't valid, delete it and create a new one. 212 * <p> 213 * If it does exist and is valid, just do some cleanup (delete temp download directories, etc.) 214 * 215 * @throws IOException if the cache folder can't be created 216 */ 217 protected void prepareCacheFolder() throws IOException { 218 locks.getCacheLock().doWriteWithLock(() -> { 219 220 if (!(cacheFolder.exists())) { 221 Utilities.createDirectory(cacheFolder.getAbsolutePath()); 222 createIniFile(); 223 } else { 224 if (!iniFileExists()) { 225 createIniFile(); 226 } 227 if (!isIniFileCurrentVersion()) { 228 clearCache(); 229 createIniFile(); 230 } 231 deleteOldTempDirectories(); 232 cleanUpCorruptPackages(); 233 } 234 return null; 235 }); 236 } 237 238 /* 239 Look for .lock files that are not actively held by a process. If found, delete the lock file, and the package 240 referenced. 241 */ 242 protected void cleanUpCorruptPackages() throws IOException { 243 for (File file : Objects.requireNonNull(cacheFolder.listFiles())) { 244 if (file.getName().endsWith(".lock")) { 245 if (locks.getCacheLock().canLockFileBeHeldByThisProcess(file)) { 246 String packageDirectoryName = file.getName().substring(0, file.getName().length() - 5); 247 log("Detected potential incomplete package installed in cache: " + packageDirectoryName + ". Attempting to delete"); 248 249 File packageDirectory = ManagedFileAccess.file(Utilities.path(cacheFolder, packageDirectoryName)); 250 if (packageDirectory.exists()) { 251 Utilities.clearDirectory(packageDirectory.getAbsolutePath()); 252 packageDirectory.delete(); 253 } 254 file.delete(); 255 log("Deleted potential incomplete package: " + packageDirectoryName); 256 } 257 } 258 } 259 } 260 261 private boolean iniFileExists() throws IOException { 262 String iniPath = getPackagesIniPath(); 263 File iniFile = ManagedFileAccess.file(iniPath); 264 return iniFile.exists(); 265 } 266 267 private boolean isIniFileCurrentVersion() throws IOException { 268 String iniPath = getPackagesIniPath(); 269 IniFile ini = new IniFile(iniPath); 270 String version = ini.getStringProperty("cache", "version"); 271 return CACHE_VERSION.equals(version); 272 } 273 274 private void deleteOldTempDirectories() throws IOException { 275 for (File f : Objects.requireNonNull(cacheFolder.listFiles())) { 276 if (f.isDirectory() && Utilities.isValidUUID(f.getName())) { 277 Utilities.clearDirectory(f.getAbsolutePath()); 278 f.delete(); 279 } 280 } 281 } 282 283 284 private void initPackageServers() { 285 myPackageServers.addAll(getConfiguredServers()); 286 if (!isIgnoreDefaultPackageServers()) { 287 myPackageServers.addAll(getDefaultServers()); 288 } 289 } 290 291 protected boolean isIgnoreDefaultPackageServers() { 292 return FhirSettings.isIgnoreDefaultPackageServers(); 293 } 294 295 @Nonnull 296 protected List<PackageServer> getDefaultServers() { 297 return PackageServer.defaultServers(); 298 } 299 300 protected List<PackageServer> getConfiguredServers() { 301 return PackageServer.getConfiguredServers(); 302 } 303 304 public String getFolder() { 305 return cacheFolder.getAbsolutePath(); 306 } 307 308 private NpmPackage loadPackageInfo(String path) throws IOException { 309 return minimalMemory ? NpmPackage.fromFolderMinimal(path, false) : NpmPackage.fromFolder(path, false); 310 } 311 312 private void clearCache() throws IOException { 313 for (File f : Objects.requireNonNull(cacheFolder.listFiles())) { 314 if (f.isDirectory()) { 315 Utilities.clearDirectory(f.getAbsolutePath()); 316 try { 317 FileUtils.deleteDirectory(f); 318 } catch (Exception e1) { 319 try { 320 FileUtils.deleteDirectory(f); 321 } catch (Exception e2) { 322 // just give up 323 } 324 } 325 326 } else if (!f.getName().equals("packages.ini")) { 327 FileUtils.forceDelete(f); 328 } 329 330 } 331 } 332 333 private void createIniFile() throws IOException { 334 IniFile ini = new IniFile(getPackagesIniPath()); 335 ini.setStringProperty("cache", "version", CACHE_VERSION, null); 336 ini.save(); 337 } 338 339 private String getPackagesIniPath() throws IOException { 340 return Utilities.path(cacheFolder, "packages.ini"); 341 } 342 343 private void checkValidVersionString(String version, String id) { 344 if (Utilities.noString(version)) { 345 throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided"); 346 } 347 if (version.startsWith("file:")) { 348 throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context"); 349 } 350 for (char ch : version.toCharArray()) { 351 if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-', '$')) { 352 throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'"); 353 } 354 } 355 } 356 357 protected InputStreamWithSrc loadFromPackageServer(String id, String version) { 358 InputStreamWithSrc retVal = super.loadFromPackageServer(id, version); 359 if (retVal != null) { 360 return retVal; 361 } 362 363 retVal = super.loadFromPackageServer(id, VersionUtilities.getMajMin(version) + ".x"); 364 if (retVal != null) { 365 return retVal; 366 } 367 368 // ok, well, we'll try the old way 369 return fetchTheOldWay(id, version); 370 } 371 372 public String getLatestVersion(String id) throws IOException { 373 for (PackageServer nextPackageServer : getPackageServers()) { 374 // special case: 375 if (!(Utilities.existsInList(id, CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageServer.PRIMARY_SERVER.equals(nextPackageServer.getUrl()))) { 376 PackageClient pc = new PackageClient(nextPackageServer); 377 try { 378 return pc.getLatestVersion(id); 379 } catch (IOException e) { 380 ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer.toString()); 381 } 382 } 383 } 384 try { 385 return fetchVersionTheOldWay(id); 386 } catch (Exception e) { 387 ourLog.info("Failed to determine latest version of package {} from server: {}", id, "build.fhir.org"); 388 } 389 // still here? use the latest version we previously found or at least, is in the cache 390 391 String version = getLatestVersionFromCache(id); 392 if (version != null) { 393 return version; 394 } 395 throw new FHIRException("Unable to find the last version for package " + id + ": no local copy, and no network access"); 396 } 397 398 public String getLatestVersionFromCache(String id) throws IOException { 399 for (String f : Utilities.reverseSorted(cacheFolder.list())) { 400 File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, f)); 401 if (cf.isDirectory()) { 402 if (f.startsWith(id + "#")) { 403 String ver = f.substring(f.indexOf("#") + 1); 404 ourLog.info("Latest version of package {} found locally is {} - using that", id, ver); 405 return ver; 406 } 407 } 408 } 409 return null; 410 } 411 412 private NpmPackage loadPackageFromFile(String id, String folder) throws IOException { 413 File f = ManagedFileAccess.file(Utilities.path(folder, id)); 414 if (!f.exists()) { 415 throw new FHIRException("Package '" + id + " not found in folder " + folder); 416 } 417 if (!f.isDirectory()) { 418 throw new FHIRException("File for '" + id + " found in folder " + folder + ", not a folder"); 419 } 420 File fp = ManagedFileAccess.file(Utilities.path(folder, id, "package", "package.json")); 421 if (!fp.exists()) { 422 throw new FHIRException("Package '" + id + " found in folder " + folder + ", but does not contain a package.json file in /package"); 423 } 424 return NpmPackage.fromFolder(f.getAbsolutePath()); 425 } 426 427 /** 428 * Clear the cache 429 * 430 * @throws IOException If the cache cannot be cleared 431 */ 432 public void clear() throws IOException { 433 this.locks.getCacheLock().doWriteWithLock(() -> { 434 clearCache(); 435 return null; 436 }); 437 } 438 439 // ========================= Utilities ============================================================================ 440 441 /** 442 * Remove a particular package from the cache 443 * 444 * @param id The id of the package to remove 445 * @param version The literal version of the package to remove. Values such as 'current' and 'dev' are not allowed. 446 * @throws IOException If the package cannot be removed 447 */ 448 public void removePackage(String id, String version) throws IOException { 449 locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> { 450 451 String f = Utilities.path(cacheFolder, id + "#" + version); 452 File ff = ManagedFileAccess.file(f); 453 if (ff.exists()) { 454 Utilities.clearDirectory(f); 455 ff.delete(); 456 } 457 458 return null; 459 }, lockParameters); 460 } 461 462 /** 463 * Load the identified package from the cache - if it exists 464 * <p/> 465 * This is for special purpose only (testing, control over speed of loading). 466 * <p/> 467 * Generally, use the loadPackage method 468 * 469 * @param id The id of the package to load 470 * @param version The version of the package to load. Values such as 'current' and 'dev' are allowed. 471 * @return The package, or null if it is not found 472 * @throws IOException If the package cannot be loaded 473 */ 474 @Override 475 public NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException { 476 477 if (!Utilities.noString(version) && version.startsWith("file:")) { 478 return loadPackageFromFile(id, version.substring(5)); 479 } 480 481 for (NpmPackage p : temporaryPackages) { 482 if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) { 483 return p; 484 } 485 if (p.name().equals(id) && Utilities.noString(version)) { 486 return p; 487 } 488 } 489 490 String foundPackageFolder = findPackageFolder(id, version); 491 if (foundPackageFolder != null) { 492 NpmPackage foundPackage = locks.getPackageLock(foundPackageFolder).doReadWithLock(() -> { 493 String path = Utilities.path(cacheFolder, foundPackageFolder); 494 File directory = ManagedFileAccess.file(path); 495 496 /* Check if the directory still exists now that we have a read lock. findPackageFolder does no locking in order 497 to avoid locking every potential package directory, so it's possible that a package deletion has occurred. 498 * */ 499 if (!directory.exists()) { 500 return null; 501 } 502 return loadPackageInfo(path); 503 }, lockParameters); 504 if (foundPackage != null) { 505 if (foundPackage.isIndexed()){ 506 return foundPackage; 507 } else { 508 return locks.getPackageLock(foundPackageFolder).doWriteWithLock(() -> { 509 File directory = ManagedFileAccess.file(foundPackage.getPath()); 510 511 /* Check if the directory still exists now that we have a write lock. findPackageFolder does no locking in order 512 to avoid locking every potential package directory, so it's possible that a package deletion has occurred. 513 * */ 514 if (!directory.exists()) { 515 return null; 516 } 517 518 // Since another thread may have already indexed the package since our read, we need to check again 519 NpmPackage output = loadPackageInfo(foundPackage.getPath()); 520 if (output.isIndexed()) { 521 return output; 522 } 523 String path = Utilities.path(cacheFolder, foundPackageFolder); 524 output.checkIndexed(path); 525 return output; 526 }, lockParameters); 527 } 528 } 529 } 530 if ("dev".equals(version)) 531 return loadPackageFromCacheOnly(id, "current"); 532 else 533 return null; 534 } 535 536 private String findPackageFolder(String id, String version) throws IOException { 537 String foundPackageFolder = null; 538 String foundVersion = null; 539 for (String currentPackageFolder : Utilities.reverseSorted(cacheFolder.list())) { 540 File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, currentPackageFolder)); 541 if (cf.isDirectory()) { 542 if (currentPackageFolder.equals(id + "#" + version) || (Utilities.noString(version) && currentPackageFolder.startsWith(id + "#"))) { 543 return currentPackageFolder; 544 } 545 if (version != null && !version.equals("current") && (version.endsWith(".x") || Utilities.charCount(version, '.') < 2) && currentPackageFolder.contains("#")) { 546 String[] parts = currentPackageFolder.split("#"); 547 if (parts[0].equals(id) && VersionUtilities.isMajMinOrLaterPatch((foundVersion != null ? foundVersion : version), parts[1])) { 548 foundVersion = parts[1]; 549 foundPackageFolder = currentPackageFolder; 550 } 551 } 552 } 553 } 554 return foundPackageFolder; 555 } 556 557 /** 558 * Add an already fetched package to the cache 559 */ 560 @Override 561 public NpmPackage addPackageToCache(final String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException { 562 checkValidVersionString(version, id); 563 return locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> { 564 String uuid = UUID.randomUUID().toString().toLowerCase(); 565 String tempDir = Utilities.path(cacheFolder, uuid); 566 567 NpmPackage npm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory); 568 569 log(""); 570 log("Installing " + id + "#" + version); 571 572 if ((npm.name() != null && id != null && !id.equalsIgnoreCase(npm.name()))) { 573 if (!suppressErrors && (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds"))) {// temporary work around 574 throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name()); 575 } 576 } 577 578 579 NpmPackage npmPackage = null; 580 String packageRoot = Utilities.path(cacheFolder, id + "#" + version); 581 try { 582 // ok, now we have a lock on it... check if something created it while we were waiting 583 if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) { 584 Utilities.createDirectory(packageRoot); 585 try { 586 Utilities.clearDirectory(packageRoot); 587 } catch (Throwable t) { 588 log("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later"); 589 } 590 Utilities.renameDirectory(tempDir, packageRoot); 591 592 log(" done."); 593 } else { 594 Utilities.clearDirectory(tempDir); 595 ManagedFileAccess.file(tempDir).delete(); 596 } 597 if (!id.equals(npm.getNpm().asString("name")) || !version.equals(npm.getNpm().asString("version"))) { 598 if (!id.equals(npm.getNpm().asString("name"))) { 599 npm.getNpm().add("original-name", npm.getNpm().asString("name")); 600 npm.getNpm().remove("name"); 601 npm.getNpm().add("name", id); 602 } 603 if (!version.equals(npm.getNpm().asString("version"))) { 604 npm.getNpm().add("original-version", npm.getNpm().asString("version")); 605 npm.getNpm().remove("version"); 606 npm.getNpm().add("version", version); 607 } 608 TextFile.stringToFile(JsonParser.compose(npm.getNpm(), true), Utilities.path(cacheFolder, id + "#" + version, "package", "package.json")); 609 } 610 npmPackage = loadPackageInfo(packageRoot); 611 if (npmPackage != null && !npmPackage.isIndexed()) { 612 npmPackage.checkIndexed(packageRoot); 613 } 614 } catch (Exception e) { 615 try { 616 // don't leave a half extracted package behind 617 log("Clean up package " + packageRoot + " because installation failed: " + e.getMessage()); 618 e.printStackTrace(); 619 Utilities.clearDirectory(packageRoot); 620 ManagedFileAccess.file(packageRoot).delete(); 621 } catch (Exception ignored) { 622 // nothing 623 } 624 throw e; 625 } 626 return npmPackage; 627 }, lockParameters); 628 } 629 630 private void log(String s) { 631 if (!silent) { 632 System.out.println(s); 633 } 634 } 635 636 @Override 637 public String getPackageUrl(String packageId) throws IOException { 638 String result = super.getPackageUrl(packageId); 639 if (result == null) { 640 result = getPackageUrlFromBuildList(packageId); 641 } 642 643 return result; 644 } 645 646 /** 647 * do not use this in minimal memory mode 648 * @param packagesFolder 649 * @throws IOException 650 */ 651 public void loadFromFolder(String packagesFolder) throws IOException { 652 assert !minimalMemory; 653 654 File[] files = ManagedFileAccess.file(packagesFolder).listFiles(); 655 if (files != null) { 656 for (File f : files) { 657 if (f.getName().endsWith(".tgz")) { 658 FileInputStream fs = ManagedFileAccess.inStream(f); 659 try { 660 temporaryPackages.add(NpmPackage.fromPackage(fs)); 661 } finally { 662 fs.close(); 663 } 664 } 665 } 666 } 667 } 668 669 @Override 670 public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException { 671 //ok, try to resolve locally 672 if (!Utilities.noString(version) && version.startsWith("file:")) { 673 return loadPackageFromFile(id, version.substring(5)); 674 } 675 676 if (version == null && id.contains("#")) { 677 version = id.substring(id.indexOf("#") + 1); 678 id = id.substring(0, id.indexOf("#")); 679 } 680 681 if (version == null) { 682 try { 683 version = getLatestVersion(id); 684 } catch (Exception e) { 685 version = null; 686 } 687 } 688 NpmPackage p = loadPackageFromCacheOnly(id, version); 689 if (p != null) { 690 if ("current".equals(version)) { 691 p = checkCurrency(id, p); 692 } 693 if (p != null) 694 return p; 695 } 696 697 if ("dev".equals(version)) { 698 p = loadPackageFromCacheOnly(id, "current"); 699 p = checkCurrency(id, p); 700 if (p != null) 701 return p; 702 version = "current"; 703 } 704 705 log("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache"); 706 log(" Fetching:"); 707 708 // nup, don't have it locally (or it's expired) 709 FilesystemPackageCacheManager.InputStreamWithSrc source; 710 if (false && packageProvider != null && packageProvider.handlesPackage(id, version)) { 711 source = packageProvider.provide(id, version); 712 } else if (Utilities.isAbsoluteUrl(version)) { 713 source = fetchSourceFromUrlSpecific(version); 714 } else if ("current".equals(version) || (version != null && version.startsWith("current$"))) { 715 // special case - fetch from ci-build server 716 source = loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null); 717 } else { 718 source = loadFromPackageServer(id, version); 719 } 720 if (source == null) { 721 throw new FHIRException("Unable to find package " + id + "#" + version); 722 } 723 return addPackageToCache(id, source.version, source.stream, source.url); 724 } 725 726 private InputStreamWithSrc fetchSourceFromUrlSpecific(String url) { 727 return new InputStreamWithSrc(fetchFromUrlSpecific(url, false), url, "current"); 728 } 729 730 private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException { 731 try { 732 HTTPResult res = ManagedWebAccess.get(source); 733 res.checkThrowException(); 734 return new ByteArrayInputStream(res.getContent()); 735 } catch (Exception e) { 736 if (optional) 737 return null; 738 else 739 throw new FHIRException("Unable to fetch: " + e.getMessage(), e); 740 } 741 } 742 743 private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException { 744 checkBuildLoaded(); 745 if (ciList.containsKey(id)) { 746 if (branch == null) { 747 InputStream stream; 748 try { 749 stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false); 750 } catch (Exception e) { 751 stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", "main", "package.tgz"), false); 752 } 753 return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current"); 754 } else { 755 InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false); 756 return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$" + branch); 757 } 758 } else if (id.startsWith("hl7.fhir.r6")) { 759 InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org", id + ".tgz"), false); 760 return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org", id + ".tgz"), "current"); 761 } else { 762 throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciList + ")"); 763 } 764 } 765 766 private String getPackageUrlFromBuildList(String packageId) throws IOException { 767 checkBuildLoaded(); 768 for (JsonObject o : buildInfo.asJsonObjects()) { 769 if (packageId.equals(o.asString("package-id"))) { 770 return o.asString("url"); 771 } 772 } 773 return null; 774 } 775 776 @Override 777 public String getPackageId(String canonicalUrl) throws IOException { 778 String retVal = findCanonicalInLocalCache(canonicalUrl); 779 780 if (retVal == null) { 781 retVal = super.getPackageId(canonicalUrl); 782 } 783 784 if (retVal == null) { 785 retVal = getPackageIdFromBuildList(canonicalUrl); 786 } 787 788 return retVal; 789 } 790 791 792 public String findCanonicalInLocalCache(String canonicalUrl) { 793 try { 794 for (String pf : listPackages()) { 795 if (ManagedFileAccess.file(Utilities.path(cacheFolder, pf, "package", "package.json")).exists()) { 796 JsonObject npm = JsonParser.parseObjectFromFile(Utilities.path(cacheFolder, pf, "package", "package.json")); 797 if (canonicalUrl.equals(npm.asString("canonical"))) { 798 return npm.asString("name"); 799 } 800 } 801 } 802 } catch (IOException e) { 803 } 804 return null; 805 } 806 807 // ========================= Package Mgmt API ======================================================================= 808 809 private String getPackageIdFromBuildList(String canonical) { 810 if (canonical == null) { 811 return null; 812 } 813 checkBuildLoaded(); 814 if (buildInfo != null) { 815 for (JsonElement n : buildInfo) { 816 JsonObject o = (JsonObject) n; 817 if (canonical.equals(o.asString("url"))) { 818 return o.asString("package-id"); 819 } 820 } 821 for (JsonElement n : buildInfo) { 822 JsonObject o = (JsonObject) n; 823 if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) { 824 return o.asString("package-id"); 825 } 826 } 827 } 828 return null; 829 } 830 831 private NpmPackage checkCurrency(String id, NpmPackage p) { 832 checkBuildLoaded(); 833 // special case: current versions roll over, and we have to check their currency 834 try { 835 String url = ciList.get(id); 836 JsonObject json = JsonParser.parseObjectFromUrl(Utilities.pathURL(url, "package.manifest.json")); 837 String currDate = json.asString("date"); 838 String packDate = p.date(); 839 if (!currDate.equals(packDate)) { 840 return null; // nup, we need a new copy 841 } 842 } catch (Exception e) { 843 log("Unable to check package currency: " + id + ": " + id); 844 } 845 return p; 846 } 847 848 private void checkBuildLoaded() { 849 if (!buildLoaded) { 850 try { 851 loadFromBuildServer(); 852 } catch (Exception e) { 853 try { 854 // we always pause a second and try again - the most common reason to be here is that the file was being changed on the server 855 Thread.sleep(1000); 856 loadFromBuildServer(); 857 } catch (Exception e2) { 858 log("Error connecting to build server - running without build (" + e2.getMessage() + ")"); 859 } 860 } 861 } 862 } 863 864 private void loadFromBuildServer() throws IOException { 865 866 HTTPResult res = ManagedWebAccess.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis()); 867 res.checkThrowException(); 868 869 buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent())); 870 871 List<BuildRecord> builds = new ArrayList<>(); 872 873 for (JsonElement n : buildInfo) { 874 JsonObject o = (JsonObject) n; 875 if (o.has("url") && o.has("package-id") && o.asString("package-id").contains(".")) { 876 String u = o.asString("url"); 877 if (u.contains("/ImplementationGuide/")) 878 u = u.substring(0, u.indexOf("/ImplementationGuide/")); 879 builds.add(new BuildRecord(u, o.asString("package-id"), getRepo(o.asString("repo")), readDate(o.asString("date")))); 880 } 881 } 882 Collections.sort(builds, new BuildRecordSorter()); 883 for (BuildRecord bld : builds) { 884 if (!ciList.containsKey(bld.getPackageId())) { 885 ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo()); 886 } 887 } 888 buildLoaded = true; 889 } 890 891 private String getRepo(String path) { 892 String[] p = path.split("\\/"); 893 return p[0] + "/" + p[1]; 894 } 895 896 private Date readDate(String s) { 897 SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US")); 898 try { 899 return sdf.parse(s); 900 } catch (ParseException e) { 901 e.printStackTrace(); 902 return new Date(); 903 } 904 } 905 906 // ----- the old way, from before package server, while everything gets onto the package server 907 private InputStreamWithSrc fetchTheOldWay(String id, String v) { 908 String url = getUrlForPackage(id); 909 if (url == null) { 910 try { 911 url = getPackageUrlFromBuildList(id); 912 } catch (Exception ignored) { 913 914 } 915 } 916 if (url == null) { 917 throw new FHIRException("Unable to resolve package id " + id + "#" + v); 918 } 919 if (url.contains("/ImplementationGuide/")) { 920 url = url.substring(0, url.indexOf("/ImplementationGuide/")); 921 } 922 String pu = Utilities.pathURL(url, "package-list.json"); 923 924 PackageList pl; 925 try { 926 pl = PackageList.fromUrl(pu); 927 } catch (Exception e) { 928 String pv = Utilities.pathURL(url, v, "package.tgz"); 929 try { 930 return new InputStreamWithSrc(fetchFromUrlSpecific(pv, false), pv, v); 931 } catch (Exception e1) { 932 throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1); 933 } 934 } 935 if (!id.equals(pl.pid())) 936 throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + pl.pid()); 937 for (PackageListEntry vo : pl.versions()) { 938 if (v.equals(vo.version())) { 939 940 String u = Utilities.pathURL(vo.path(), "package.tgz"); 941 return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v); 942 } 943 } 944 945 return null; 946 } 947 948 949 // ---------- Current Build SubSystem -------------------------------------------------------------------------------------- 950 951 private String fetchVersionTheOldWay(String id) throws IOException { 952 String url = getUrlForPackage(id); 953 if (url == null) { 954 try { 955 url = getPackageUrlFromBuildList(id); 956 } catch (Exception e) { 957 url = null; 958 } 959 } 960 if (url == null) { 961 throw new FHIRException("Unable to resolve package id " + id); 962 } 963 PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json")); 964 if (!id.equals(pl.pid())) 965 throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid()); 966 for (PackageListEntry vo : pl.versions()) { 967 if (vo.current()) { 968 return vo.version(); 969 } 970 } 971 972 return null; 973 } 974 975 private String getUrlForPackage(String id) { 976 if (CommonPackages.ID_XVER.equals(id)) { 977 return "https://fhir.org/packages/hl7.fhir.xver-extensions"; 978 } 979 return null; 980 } 981 982 public List<String> listPackages() { 983 List<String> res = new ArrayList<>(); 984 for (File f : cacheFolder.listFiles()) { 985 if (f.isDirectory() && f.getName().contains("#")) { 986 res.add(f.getName()); 987 } 988 } 989 return res; 990 } 991 992 public interface CacheLockFunction<T> { 993 T get() throws IOException; 994 } 995 996 public class BuildRecordSorter implements Comparator<BuildRecord> { 997 998 @Override 999 public int compare(BuildRecord arg0, BuildRecord arg1) { 1000 return arg1.date.compareTo(arg0.date); 1001 } 1002 } 1003 1004 public class BuildRecord { 1005 1006 private final String url; 1007 private final String packageId; 1008 private final String repo; 1009 private final Date date; 1010 1011 public BuildRecord(String url, String packageId, String repo, Date date) { 1012 super(); 1013 this.url = url; 1014 this.packageId = packageId; 1015 this.repo = repo; 1016 this.date = date; 1017 } 1018 1019 public String getUrl() { 1020 return url; 1021 } 1022 1023 public String getPackageId() { 1024 return packageId; 1025 } 1026 1027 public String getRepo() { 1028 return repo; 1029 } 1030 1031 public Date getDate() { 1032 return date; 1033 } 1034 1035 } 1036 1037 public boolean packageExists(String id, String ver) throws IOException { 1038 if (packageInstalled(id, ver)) { 1039 return true; 1040 } 1041 for (PackageServer s : getPackageServers()) { 1042 if (new PackageClient(s).exists(id, ver)) { 1043 return true; 1044 } 1045 } 1046 return false; 1047 } 1048 1049 public boolean packageInstalled(String id, String version) { 1050 for (NpmPackage p : temporaryPackages) { 1051 if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) { 1052 return true; 1053 } 1054 if (p.name().equals(id) && Utilities.noString(version)) { 1055 return true; 1056 } 1057 } 1058 1059 for (String f : Utilities.sorted(cacheFolder.list())) { 1060 if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) { 1061 return true; 1062 } 1063 } 1064 if ("dev".equals(version)) 1065 return packageInstalled(id, "current"); 1066 else 1067 return false; 1068 } 1069 1070 public boolean isSuppressErrors() { 1071 return suppressErrors; 1072 } 1073 1074 public void setSuppressErrors(boolean suppressErrors) { 1075 this.suppressErrors = suppressErrors; 1076 } 1077 1078 public static IPackageProvider getPackageProvider() { 1079 return packageProvider; 1080 } 1081 1082 public static void setPackageProvider(IPackageProvider packageProvider) { 1083 FilesystemPackageCacheManager.packageProvider = packageProvider; 1084 } 1085 1086 1087}