001package org.hl7.fhir.utilities.npm;
002
003import java.io.*;
004
005import java.text.ParseException;
006import java.text.SimpleDateFormat;
007
008import java.util.*;
009
010import javax.annotation.Nonnull;
011import javax.annotation.Nullable;
012
013import lombok.Getter;
014import lombok.Setter;
015import lombok.With;
016import org.apache.commons.io.FileUtils;
017import org.hl7.fhir.exceptions.FHIRException;
018import org.hl7.fhir.utilities.IniFile;
019import org.hl7.fhir.utilities.TextFile;
020import org.hl7.fhir.utilities.Utilities;
021import org.hl7.fhir.utilities.VersionUtilities;
022import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
023import org.hl7.fhir.utilities.http.HTTPResult;
024import org.hl7.fhir.utilities.http.ManagedWebAccess;
025import org.hl7.fhir.utilities.json.model.JsonArray;
026import org.hl7.fhir.utilities.json.model.JsonElement;
027import org.hl7.fhir.utilities.json.model.JsonObject;
028import org.hl7.fhir.utilities.json.parser.JsonParser;
029import org.hl7.fhir.utilities.npm.PackageList.PackageListEntry;
030import org.hl7.fhir.utilities.settings.FhirSettings;
031import org.slf4j.Logger;
032import org.slf4j.LoggerFactory;
033
034/*
035  Copyright (c) 2011+, HL7, Inc.
036  All rights reserved.
037  
038  Redistribution and use in source and binary forms, with or without modification, 
039  are permitted provided that the following conditions are met:
040    
041   * Redistributions of source code must retain the above copyright notice, this 
042     list of conditions and the following disclaimer.
043   * Redistributions in binary form must reproduce the above copyright notice, 
044     this list of conditions and the following disclaimer in the documentation 
045     and/or other materials provided with the distribution.
046   * Neither the name of HL7 nor the names of its contributors may be used to 
047     endorse or promote products derived from this software without specific 
048     prior written permission.
049  
050  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 
051  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 
052  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 
053  IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 
054  INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 
055  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 
056  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
057  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 
058  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 
059  POSSIBILITY OF SUCH DAMAGE.
060  
061 */
062
063/**
064 * This is a package cache manager implementation that uses a local disk cache
065 *
066 * <p>
067 * API:
068 * <p>
069 * constructor
070 * getPackageUrl
071 * getPackageId
072 * findPackageCache
073 * addPackageToCache
074 *
075 * @author Grahame Grieve
076 */
077public class FilesystemPackageCacheManager extends BasePackageCacheManager implements IPackageCacheManager {
078
079  private final FilesystemPackageCacheManagerLocks locks;
080
081  private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters;
082
083  // When running in testing mode, some packages are provided from the test case repository rather than by the normal means
084  // the PackageProvider is responsible for this. if no package provider is defined, or it declines to handle the package, 
085  // then the normal means will be used
086  public interface IPackageProvider {
087    boolean handlesPackage(String id, String version);
088    InputStreamWithSrc provide(String id, String version) throws IOException;
089  }
090
091  private static IPackageProvider packageProvider;
092  public static final String PACKAGE_REGEX = "^[a-zA-Z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+$";
093  public static final String PACKAGE_VERSION_REGEX = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+\\#[A-Za-z0-9\\-\\_\\$]+(\\.[A-Za-z0-9\\-\\_\\$]+)*$";
094  public static final String PACKAGE_VERSION_REGEX_OPT = "^[A-Za-z][A-Za-z0-9\\_\\-]*(\\.[A-Za-z0-9\\_\\-]+)+(\\#[A-Za-z0-9\\-\\_]+(\\.[A-Za-z0-9\\-\\_]+)*)?$";
095  private static final Logger ourLog = LoggerFactory.getLogger(FilesystemPackageCacheManager.class);
096  private static final String CACHE_VERSION = "3"; // second version - see wiki page
097
098  @Nonnull
099  private final File cacheFolder;
100
101  private final List<NpmPackage> temporaryPackages = new ArrayList<>();
102  private boolean buildLoaded = false;
103  private final Map<String, String> ciList = new HashMap<>();
104  private JsonArray buildInfo;
105  private boolean suppressErrors;
106
107  @Setter
108  @Getter
109  private boolean minimalMemory;
110
111  public static class Builder {
112
113    @Getter
114    private final File cacheFolder;
115
116    @With
117    @Getter
118    private final List<PackageServer> packageServers;
119
120    @With
121    @Getter
122    private final FilesystemPackageCacheManagerLocks.LockParameters lockParameters;
123
124    public Builder() throws IOException {
125      this.cacheFolder = getUserCacheFolder();
126      this.packageServers = getPackageServersFromFHIRSettings();
127      this.lockParameters = null;
128    }
129
130    private Builder(File cacheFolder, List<PackageServer> packageServers, FilesystemPackageCacheManagerLocks.LockParameters lockParameters) {
131      this.cacheFolder = cacheFolder;
132      this.packageServers = packageServers;
133      this.lockParameters = lockParameters;
134    }
135
136    private File getUserCacheFolder() throws IOException {
137      return ManagedFileAccess.file(Utilities.path(System.getProperty("user.home"), ".fhir", "packages"));
138    }
139
140    private List<PackageServer> getPackageServersFromFHIRSettings() {
141      List<PackageServer> packageServers = new ArrayList<>(getConfiguredServers());
142      if (!isIgnoreDefaultPackageServers()) {
143        packageServers.addAll(getDefaultServers());
144      }
145      return packageServers;
146    }
147
148    protected boolean isIgnoreDefaultPackageServers() {
149      return FhirSettings.isIgnoreDefaultPackageServers();
150    }
151
152    @Nonnull
153    protected List<PackageServer> getDefaultServers() {
154      return PackageServer.defaultServers();
155    }
156
157    protected List<PackageServer> getConfiguredServers() {
158      return PackageServer.getConfiguredServers();
159    }
160
161    public Builder withCacheFolder(String cacheFolderPath) throws IOException {
162      File cacheFolder = ManagedFileAccess.file(cacheFolderPath);
163      if (!cacheFolder.exists()) {
164        throw new FHIRException("The folder '" + cacheFolder + "' could not be found");
165      }
166      return new Builder(cacheFolder, this.packageServers, this.lockParameters);
167    }
168
169    public Builder withSystemCacheFolder() throws IOException {
170      final File systemCacheFolder;
171      if (Utilities.isWindows()) {
172        systemCacheFolder = ManagedFileAccess.file(Utilities.path(System.getenv("ProgramData"), ".fhir", "packages"));
173      } else {
174        systemCacheFolder = ManagedFileAccess.file(Utilities.path("/var", "lib", ".fhir", "packages"));
175      }
176      return new Builder(systemCacheFolder, this.packageServers, this.lockParameters);
177    }
178
179    public Builder withTestingCacheFolder() throws IOException {
180      return new Builder(ManagedFileAccess.file(Utilities.path("[tmp]", ".fhir", "packages")), this.packageServers, this.lockParameters);
181    }
182
183    public FilesystemPackageCacheManager build() throws IOException {
184      final FilesystemPackageCacheManagerLocks locks;
185      try {
186        locks = FilesystemPackageCacheManagerLocks.getFilesystemPackageCacheManagerLocks(cacheFolder);
187      } catch (RuntimeException e) {
188        if (e.getCause() instanceof IOException) {
189          throw (IOException) e.getCause();
190        } else {
191          throw e;
192        }
193      }
194      return new FilesystemPackageCacheManager(cacheFolder, packageServers, locks, lockParameters);
195    }
196  }
197
198  private FilesystemPackageCacheManager(@Nonnull File cacheFolder, @Nonnull List<PackageServer> packageServers, @Nonnull FilesystemPackageCacheManagerLocks locks, @Nullable FilesystemPackageCacheManagerLocks.LockParameters lockParameters) throws IOException {
199    super(packageServers);
200    this.cacheFolder = cacheFolder;
201    this.locks = locks;
202    this.lockParameters = lockParameters;
203    prepareCacheFolder();
204  }
205
206  /**
207   * Check if the cache folder exists and is valid.
208   * <p>
209   * If it doesn't exist, create it.
210   * <p>
211   * If it does exist and isn't valid, delete it and create a new one.
212   * <p>
213   * If it does exist and is valid, just do some cleanup (delete temp download directories, etc.)
214   *
215   * @throws IOException if the cache folder can't be created
216   */
217  protected void prepareCacheFolder() throws IOException {
218    locks.getCacheLock().doWriteWithLock(() -> {
219
220      if (!(cacheFolder.exists())) {
221        Utilities.createDirectory(cacheFolder.getAbsolutePath());
222        createIniFile();
223      } else {
224        if (!iniFileExists()) {
225          createIniFile();
226        }
227        if (!isIniFileCurrentVersion()) {
228          clearCache();
229          createIniFile();
230        }
231        deleteOldTempDirectories();
232        cleanUpCorruptPackages();
233      }
234      return null;
235    });
236  }
237
238  /*
239    Look for .lock files that are not actively held by a process. If found, delete the lock file, and the package
240    referenced.
241   */
242  protected void cleanUpCorruptPackages() throws IOException {
243    for (File file : Objects.requireNonNull(cacheFolder.listFiles())) {
244      if (file.getName().endsWith(".lock")) {
245        if (locks.getCacheLock().canLockFileBeHeldByThisProcess(file)) {
246          String packageDirectoryName = file.getName().substring(0, file.getName().length() - 5);
247          log("Detected potential incomplete package installed in cache: " + packageDirectoryName + ". Attempting to delete");
248
249          File packageDirectory = ManagedFileAccess.file(Utilities.path(cacheFolder, packageDirectoryName));
250          if (packageDirectory.exists()) {
251            Utilities.clearDirectory(packageDirectory.getAbsolutePath());
252            packageDirectory.delete();
253          }
254          file.delete();
255          log("Deleted potential incomplete package: " + packageDirectoryName);
256        }
257      }
258    }
259  }
260
261  private boolean iniFileExists() throws IOException {
262    String iniPath = getPackagesIniPath();
263    File iniFile = ManagedFileAccess.file(iniPath);
264    return iniFile.exists();
265  }
266
267  private boolean isIniFileCurrentVersion() throws IOException {
268    String iniPath = getPackagesIniPath();
269    IniFile ini = new IniFile(iniPath);
270    String version = ini.getStringProperty("cache", "version");
271    return CACHE_VERSION.equals(version);
272  }
273
274  private void deleteOldTempDirectories() throws IOException {
275    for (File f : Objects.requireNonNull(cacheFolder.listFiles())) {
276      if (f.isDirectory() && Utilities.isValidUUID(f.getName())) {
277        Utilities.clearDirectory(f.getAbsolutePath());
278        f.delete();
279      }
280    }
281  }
282
283
284  private void initPackageServers() {
285    myPackageServers.addAll(getConfiguredServers());
286    if (!isIgnoreDefaultPackageServers()) {
287      myPackageServers.addAll(getDefaultServers());
288    }
289  }
290
291  protected boolean isIgnoreDefaultPackageServers() {
292    return FhirSettings.isIgnoreDefaultPackageServers();
293  }
294
295  @Nonnull
296  protected List<PackageServer> getDefaultServers() {
297    return PackageServer.defaultServers();
298  }
299
300  protected List<PackageServer> getConfiguredServers() {
301    return PackageServer.getConfiguredServers();
302  }
303
304  public String getFolder() {
305    return cacheFolder.getAbsolutePath();
306  }
307
308  private NpmPackage loadPackageInfo(String path) throws IOException {
309    return minimalMemory ? NpmPackage.fromFolderMinimal(path, false) : NpmPackage.fromFolder(path, false);
310  }
311
312  private void clearCache() throws IOException {
313    for (File f : Objects.requireNonNull(cacheFolder.listFiles())) {
314      if (f.isDirectory()) {
315        Utilities.clearDirectory(f.getAbsolutePath());
316        try {
317          FileUtils.deleteDirectory(f);
318        } catch (Exception e1) {
319          try {
320            FileUtils.deleteDirectory(f);
321          } catch (Exception e2) {
322            // just give up
323          }
324        }
325
326      } else if (!f.getName().equals("packages.ini")) {
327        FileUtils.forceDelete(f);
328      }
329
330    }
331  }
332
333  private void createIniFile() throws IOException {
334    IniFile ini = new IniFile(getPackagesIniPath());
335    ini.setStringProperty("cache", "version", CACHE_VERSION, null);
336    ini.save();
337  }
338
339  private String getPackagesIniPath() throws IOException {
340    return Utilities.path(cacheFolder, "packages.ini");
341  }
342
343  private void checkValidVersionString(String version, String id) {
344    if (Utilities.noString(version)) {
345      throw new FHIRException("Cannot add package " + id + " to the package cache - a version must be provided");
346    }
347    if (version.startsWith("file:")) {
348      throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal in this context");
349    }
350    for (char ch : version.toCharArray()) {
351      if (!Character.isAlphabetic(ch) && !Character.isDigit(ch) && !Utilities.existsInList(ch, '.', '-', '$')) {
352        throw new FHIRException("Cannot add package " + id + " to the package cache - the version '" + version + "' is illegal (ch '" + ch + "'");
353      }
354    }
355  }
356
357  protected InputStreamWithSrc loadFromPackageServer(String id, String version) {
358    InputStreamWithSrc retVal = super.loadFromPackageServer(id, version);
359    if (retVal != null) {
360      return retVal;
361    }
362
363    retVal = super.loadFromPackageServer(id, VersionUtilities.getMajMin(version) + ".x");
364    if (retVal != null) {
365      return retVal;
366    }
367
368    // ok, well, we'll try the old way
369    return fetchTheOldWay(id, version);
370  }
371
372  public String getLatestVersion(String id) throws IOException {
373    for (PackageServer nextPackageServer : getPackageServers()) {
374      // special case:
375      if (!(Utilities.existsInList(id, CommonPackages.ID_PUBPACK, "hl7.terminology.r5") && PackageServer.PRIMARY_SERVER.equals(nextPackageServer.getUrl()))) {
376        PackageClient pc = new PackageClient(nextPackageServer);
377        try {
378          return pc.getLatestVersion(id);
379        } catch (IOException e) {
380          ourLog.info("Failed to determine latest version of package {} from server: {}", id, nextPackageServer.toString());
381        }
382      }
383    }
384    try {
385      return fetchVersionTheOldWay(id);
386    } catch (Exception e) {
387      ourLog.info("Failed to determine latest version of package {} from server: {}", id, "build.fhir.org");
388    }
389    // still here? use the latest version we previously found or at least, is in the cache
390
391    String version = getLatestVersionFromCache(id);
392    if (version != null) {
393      return version;
394    }
395    throw new FHIRException("Unable to find the last version for package " + id + ": no local copy, and no network access");
396  }
397
398  public String getLatestVersionFromCache(String id) throws IOException {
399    for (String f : Utilities.reverseSorted(cacheFolder.list())) {
400      File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, f));
401      if (cf.isDirectory()) {
402        if (f.startsWith(id + "#")) {
403          String ver = f.substring(f.indexOf("#") + 1);
404          ourLog.info("Latest version of package {} found locally is {} - using that", id, ver);
405          return ver;
406        }
407      }
408    }
409    return null;
410  }
411
412  private NpmPackage loadPackageFromFile(String id, String folder) throws IOException {
413    File f = ManagedFileAccess.file(Utilities.path(folder, id));
414    if (!f.exists()) {
415      throw new FHIRException("Package '" + id + "  not found in folder " + folder);
416    }
417    if (!f.isDirectory()) {
418      throw new FHIRException("File for '" + id + "  found in folder " + folder + ", not a folder");
419    }
420    File fp = ManagedFileAccess.file(Utilities.path(folder, id, "package", "package.json"));
421    if (!fp.exists()) {
422      throw new FHIRException("Package '" + id + "  found in folder " + folder + ", but does not contain a package.json file in /package");
423    }
424    return NpmPackage.fromFolder(f.getAbsolutePath());
425  }
426
427  /**
428   * Clear the cache
429   *
430   * @throws IOException If the cache cannot be cleared
431   */
432  public void clear() throws IOException {
433    this.locks.getCacheLock().doWriteWithLock(() -> {
434      clearCache();
435      return null;
436    });
437  }
438
439  // ========================= Utilities ============================================================================
440
441  /**
442   * Remove a particular package from the cache
443   *
444   * @param id The id of the package to remove
445   * @param version The literal version of the package to remove. Values such as 'current' and 'dev' are not allowed.
446   * @throws IOException If the package cannot be removed
447   */
448  public void removePackage(String id, String version) throws IOException {
449    locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> {
450
451      String f = Utilities.path(cacheFolder, id + "#" + version);
452      File ff = ManagedFileAccess.file(f);
453      if (ff.exists()) {
454        Utilities.clearDirectory(f);
455        ff.delete();
456      }
457
458      return null;
459    }, lockParameters);
460  }
461
462  /**
463   * Load the identified package from the cache - if it exists
464   * <p/>
465   * This is for special purpose only (testing, control over speed of loading).
466   * <p/>
467   * Generally, use the loadPackage method
468   *
469   * @param id The id of the package to load
470   * @param version The version of the package to load. Values such as 'current' and 'dev' are allowed.
471   * @return The package, or null if it is not found
472   * @throws IOException If the package cannot be loaded
473   */
474  @Override
475  public NpmPackage loadPackageFromCacheOnly(String id, @Nullable String version) throws IOException {
476
477    if (!Utilities.noString(version) && version.startsWith("file:")) {
478      return loadPackageFromFile(id, version.substring(5));
479    }
480
481    for (NpmPackage p : temporaryPackages) {
482      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
483        return p;
484      }
485      if (p.name().equals(id) && Utilities.noString(version)) {
486        return p;
487      }
488    }
489
490    String foundPackageFolder = findPackageFolder(id, version);
491    if (foundPackageFolder != null) {
492      NpmPackage foundPackage = locks.getPackageLock(foundPackageFolder).doReadWithLock(() -> {
493        String path = Utilities.path(cacheFolder, foundPackageFolder);
494        File directory = ManagedFileAccess.file(path);
495
496        /* Check if the directory still exists now that we have a read lock. findPackageFolder does no locking in order
497        to avoid locking every potential package directory, so it's possible that a package deletion has occurred.
498        * */
499        if (!directory.exists()) {
500          return null;
501        }
502        return loadPackageInfo(path);
503      }, lockParameters);
504      if (foundPackage != null) {
505        if (foundPackage.isIndexed()){
506          return foundPackage;
507        } else {
508          return locks.getPackageLock(foundPackageFolder).doWriteWithLock(() -> {
509            File directory = ManagedFileAccess.file(foundPackage.getPath());
510
511        /* Check if the directory still exists now that we have a write lock. findPackageFolder does no locking in order
512        to avoid locking every potential package directory, so it's possible that a package deletion has occurred.
513        * */
514            if (!directory.exists()) {
515              return null;
516            }
517
518            // Since another thread may have already indexed the package since our read, we need to check again
519            NpmPackage output = loadPackageInfo(foundPackage.getPath());
520            if (output.isIndexed()) {
521              return output;
522            }
523            String path = Utilities.path(cacheFolder, foundPackageFolder);
524            output.checkIndexed(path);
525            return output;
526          }, lockParameters);
527          }
528      }
529    }
530    if ("dev".equals(version))
531      return loadPackageFromCacheOnly(id, "current");
532    else
533      return null;
534  }
535
536  private String findPackageFolder(String id, String version) throws IOException {
537    String foundPackageFolder = null;
538    String foundVersion = null;
539    for (String currentPackageFolder : Utilities.reverseSorted(cacheFolder.list())) {
540      File cf = ManagedFileAccess.file(Utilities.path(cacheFolder, currentPackageFolder));
541      if (cf.isDirectory()) {
542        if (currentPackageFolder.equals(id + "#" + version) || (Utilities.noString(version) && currentPackageFolder.startsWith(id + "#"))) {
543          return currentPackageFolder;
544        }
545        if (version != null && !version.equals("current") && (version.endsWith(".x") || Utilities.charCount(version, '.') < 2) && currentPackageFolder.contains("#")) {
546          String[] parts = currentPackageFolder.split("#");
547          if (parts[0].equals(id) && VersionUtilities.isMajMinOrLaterPatch((foundVersion != null ? foundVersion : version), parts[1])) {
548            foundVersion = parts[1];
549            foundPackageFolder = currentPackageFolder;
550          }
551        }
552      }
553    }
554    return foundPackageFolder;
555  }
556
557  /**
558   * Add an already fetched package to the cache
559   */
560  @Override
561  public NpmPackage addPackageToCache(final String id, final String version, final InputStream packageTgzInputStream, final String sourceDesc) throws IOException {
562    checkValidVersionString(version, id);
563    return locks.getPackageLock(id + "#" + version).doWriteWithLock(() -> {
564      String uuid = UUID.randomUUID().toString().toLowerCase();
565      String tempDir = Utilities.path(cacheFolder, uuid);
566
567      NpmPackage npm = NpmPackage.extractFromTgz(packageTgzInputStream, sourceDesc, tempDir, minimalMemory);
568
569      log("");
570      log("Installing " + id + "#" + version);
571
572      if ((npm.name() != null && id != null && !id.equalsIgnoreCase(npm.name()))) {
573        if (!suppressErrors && (!id.equals("hl7.fhir.r5.core") && !id.equals("hl7.fhir.us.immds"))) {// temporary work around
574          throw new IOException("Attempt to import a mis-identified package. Expected " + id + ", got " + npm.name());
575        }
576      }
577
578
579      NpmPackage npmPackage = null;
580      String packageRoot = Utilities.path(cacheFolder, id + "#" + version);
581      try {
582        // ok, now we have a lock on it... check if something created it while we were waiting
583        if (!ManagedFileAccess.file(packageRoot).exists() || Utilities.existsInList(version, "current", "dev")) {
584          Utilities.createDirectory(packageRoot);
585          try {
586            Utilities.clearDirectory(packageRoot);
587          } catch (Throwable t) {
588            log("Unable to clear directory: " + packageRoot + ": " + t.getMessage() + " - this may cause problems later");
589          }
590          Utilities.renameDirectory(tempDir, packageRoot);
591
592          log(" done.");
593        } else {
594          Utilities.clearDirectory(tempDir);
595          ManagedFileAccess.file(tempDir).delete();
596        }
597        if (!id.equals(npm.getNpm().asString("name")) || !version.equals(npm.getNpm().asString("version"))) {
598          if (!id.equals(npm.getNpm().asString("name"))) {
599            npm.getNpm().add("original-name", npm.getNpm().asString("name"));
600            npm.getNpm().remove("name");
601            npm.getNpm().add("name", id);
602          }
603          if (!version.equals(npm.getNpm().asString("version"))) {
604            npm.getNpm().add("original-version", npm.getNpm().asString("version"));
605            npm.getNpm().remove("version");
606            npm.getNpm().add("version", version);
607          }
608          TextFile.stringToFile(JsonParser.compose(npm.getNpm(), true), Utilities.path(cacheFolder, id + "#" + version, "package", "package.json"));
609        }
610        npmPackage = loadPackageInfo(packageRoot);
611        if (npmPackage != null && !npmPackage.isIndexed()) {
612          npmPackage.checkIndexed(packageRoot);
613        }
614      } catch (Exception e) {
615        try {
616          // don't leave a half extracted package behind
617          log("Clean up package " + packageRoot + " because installation failed: " + e.getMessage());
618          e.printStackTrace();
619          Utilities.clearDirectory(packageRoot);
620          ManagedFileAccess.file(packageRoot).delete();
621        } catch (Exception ignored) {
622          // nothing
623        }
624        throw e;
625      }
626      return npmPackage;
627    }, lockParameters);
628  }
629
630  private void log(String s) {
631    if (!silent) {
632      System.out.println(s);
633    }
634  }
635
636  @Override
637  public String getPackageUrl(String packageId) throws IOException {
638    String result = super.getPackageUrl(packageId);
639    if (result == null) {
640      result = getPackageUrlFromBuildList(packageId);
641    }
642
643    return result;
644  }
645
646  /**
647   * do not use this in minimal memory mode
648   * @param packagesFolder
649   * @throws IOException
650   */
651  public void loadFromFolder(String packagesFolder) throws IOException {
652    assert !minimalMemory;
653
654    File[] files = ManagedFileAccess.file(packagesFolder).listFiles();
655    if (files != null) {
656      for (File f : files) {
657        if (f.getName().endsWith(".tgz")) {
658          FileInputStream fs = ManagedFileAccess.inStream(f);
659          try {
660            temporaryPackages.add(NpmPackage.fromPackage(fs));
661          } finally {
662            fs.close();
663          }
664        }
665      }
666    }
667  }
668
669  @Override
670  public NpmPackage loadPackage(String id, String version) throws FHIRException, IOException {
671    //ok, try to resolve locally
672    if (!Utilities.noString(version) && version.startsWith("file:")) {
673      return loadPackageFromFile(id, version.substring(5));
674    }
675
676    if (version == null && id.contains("#")) {
677      version = id.substring(id.indexOf("#") + 1);
678      id = id.substring(0, id.indexOf("#"));
679    }
680
681    if (version == null) {
682      try {
683        version = getLatestVersion(id);
684      } catch (Exception e) {
685        version = null;
686      }
687    }
688    NpmPackage p = loadPackageFromCacheOnly(id, version);
689    if (p != null) {
690      if ("current".equals(version)) {
691        p = checkCurrency(id, p);
692      }
693      if (p != null)
694        return p;
695    }
696
697    if ("dev".equals(version)) {
698      p = loadPackageFromCacheOnly(id, "current");
699      p = checkCurrency(id, p);
700      if (p != null)
701        return p;
702      version = "current";
703    }
704
705    log("Installing " + id + "#" + (version == null ? "?" : version) + " to the package cache");
706    log("  Fetching:");
707
708    // nup, don't have it locally (or it's expired)
709    FilesystemPackageCacheManager.InputStreamWithSrc source;
710    if (false && packageProvider != null && packageProvider.handlesPackage(id, version)) {
711      source = packageProvider.provide(id, version);
712    } else if (Utilities.isAbsoluteUrl(version)) {
713      source = fetchSourceFromUrlSpecific(version);
714    } else if ("current".equals(version) || (version != null && version.startsWith("current$"))) {
715      // special case - fetch from ci-build server
716      source = loadFromCIBuild(id, version.startsWith("current$") ? version.substring(8) : null);
717    } else {
718      source = loadFromPackageServer(id, version);
719    }
720    if (source == null) {
721      throw new FHIRException("Unable to find package " + id + "#" + version);
722    }
723    return addPackageToCache(id, source.version, source.stream, source.url);
724  }
725
726  private InputStreamWithSrc fetchSourceFromUrlSpecific(String url) {
727    return new InputStreamWithSrc(fetchFromUrlSpecific(url, false), url, "current");
728  }
729
730  private InputStream fetchFromUrlSpecific(String source, boolean optional) throws FHIRException {
731    try {
732      HTTPResult res = ManagedWebAccess.get(source);
733      res.checkThrowException();
734      return new ByteArrayInputStream(res.getContent());
735    } catch (Exception e) {
736      if (optional)
737        return null;
738      else
739        throw new FHIRException("Unable to fetch: " + e.getMessage(), e);
740    }
741  }
742
743  private InputStreamWithSrc loadFromCIBuild(String id, String branch) throws IOException {
744    checkBuildLoaded();
745    if (ciList.containsKey(id)) {
746      if (branch == null) {
747        InputStream stream;
748        try {
749          stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "package.tgz"), false);
750        } catch (Exception e) {
751          stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", "main", "package.tgz"), false);
752        }
753        return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "package.tgz"), "current");
754      } else {
755        InputStream stream = fetchFromUrlSpecific(Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), false);
756        return new InputStreamWithSrc(stream, Utilities.pathURL(ciList.get(id), "branches", branch, "package.tgz"), "current$" + branch);
757      }
758    } else if (id.startsWith("hl7.fhir.r6")) {
759      InputStream stream = fetchFromUrlSpecific(Utilities.pathURL("https://build.fhir.org", id + ".tgz"), false);
760      return new InputStreamWithSrc(stream, Utilities.pathURL("https://build.fhir.org", id + ".tgz"), "current");
761    } else {
762      throw new FHIRException("The package '" + id + "' has no entry on the current build server (" + ciList + ")");
763    }
764  }
765
766  private String getPackageUrlFromBuildList(String packageId) throws IOException {
767    checkBuildLoaded();
768    for (JsonObject o : buildInfo.asJsonObjects()) {
769      if (packageId.equals(o.asString("package-id"))) {
770        return o.asString("url");
771      }
772    }
773    return null;
774  }
775
776  @Override
777  public String getPackageId(String canonicalUrl) throws IOException {
778    String retVal = findCanonicalInLocalCache(canonicalUrl);
779
780    if (retVal == null) {
781      retVal = super.getPackageId(canonicalUrl);
782    }
783
784    if (retVal == null) {
785      retVal = getPackageIdFromBuildList(canonicalUrl);
786    }
787
788    return retVal;
789  }
790
791
792  public String findCanonicalInLocalCache(String canonicalUrl) {
793    try {
794      for (String pf : listPackages()) {
795        if (ManagedFileAccess.file(Utilities.path(cacheFolder, pf, "package", "package.json")).exists()) {
796          JsonObject npm = JsonParser.parseObjectFromFile(Utilities.path(cacheFolder, pf, "package", "package.json"));
797          if (canonicalUrl.equals(npm.asString("canonical"))) {
798            return npm.asString("name");
799          }
800        }
801      }
802    } catch (IOException e) {
803    }
804    return null;
805  }
806
807  // ========================= Package Mgmt API =======================================================================
808
809  private String getPackageIdFromBuildList(String canonical) {
810    if (canonical == null) {
811      return null;
812    }
813    checkBuildLoaded();
814    if (buildInfo != null) {
815      for (JsonElement n : buildInfo) {
816        JsonObject o = (JsonObject) n;
817        if (canonical.equals(o.asString("url"))) {
818          return o.asString("package-id");
819        }
820      }
821      for (JsonElement n : buildInfo) {
822        JsonObject o = (JsonObject) n;
823        if (o.asString("url").startsWith(canonical + "/ImplementationGuide/")) {
824          return o.asString("package-id");
825        }
826      }
827    }
828    return null;
829  }
830
831  private NpmPackage checkCurrency(String id, NpmPackage p) {
832    checkBuildLoaded();
833    // special case: current versions roll over, and we have to check their currency
834    try {
835      String url = ciList.get(id);
836      JsonObject json = JsonParser.parseObjectFromUrl(Utilities.pathURL(url, "package.manifest.json"));
837      String currDate = json.asString("date");
838      String packDate = p.date();
839      if (!currDate.equals(packDate)) {
840        return null; // nup, we need a new copy
841      }
842    } catch (Exception e) {
843      log("Unable to check package currency: " + id + ": " + id);
844    }
845    return p;
846  }
847
848  private void checkBuildLoaded() {
849    if (!buildLoaded) {
850      try {
851        loadFromBuildServer();
852      } catch (Exception e) {
853        try {
854          // we always pause a second and try again - the most common reason to be here is that the file was being changed on the server
855          Thread.sleep(1000);
856          loadFromBuildServer();
857        } catch (Exception e2) {
858          log("Error connecting to build server - running without build (" + e2.getMessage() + ")");
859        }
860      }
861    }
862  }
863
864  private void loadFromBuildServer() throws IOException {
865
866    HTTPResult res = ManagedWebAccess.get("https://build.fhir.org/ig/qas.json?nocache=" + System.currentTimeMillis());
867    res.checkThrowException();
868
869    buildInfo = (JsonArray) JsonParser.parse(TextFile.bytesToString(res.getContent()));
870
871    List<BuildRecord> builds = new ArrayList<>();
872
873    for (JsonElement n : buildInfo) {
874      JsonObject o = (JsonObject) n;
875      if (o.has("url") && o.has("package-id") && o.asString("package-id").contains(".")) {
876        String u = o.asString("url");
877        if (u.contains("/ImplementationGuide/"))
878          u = u.substring(0, u.indexOf("/ImplementationGuide/"));
879        builds.add(new BuildRecord(u, o.asString("package-id"), getRepo(o.asString("repo")), readDate(o.asString("date"))));
880      }
881    }
882    Collections.sort(builds, new BuildRecordSorter());
883    for (BuildRecord bld : builds) {
884      if (!ciList.containsKey(bld.getPackageId())) {
885        ciList.put(bld.getPackageId(), "https://build.fhir.org/ig/" + bld.getRepo());
886      }
887    }
888    buildLoaded = true;
889  }
890
891  private String getRepo(String path) {
892    String[] p = path.split("\\/");
893    return p[0] + "/" + p[1];
894  }
895
896  private Date readDate(String s) {
897    SimpleDateFormat sdf = new SimpleDateFormat("EEE, dd MMM, yyyy HH:mm:ss Z", new Locale("en", "US"));
898    try {
899      return sdf.parse(s);
900    } catch (ParseException e) {
901      e.printStackTrace();
902      return new Date();
903    }
904  }
905
906  // ----- the old way, from before package server, while everything gets onto the package server
907  private InputStreamWithSrc fetchTheOldWay(String id, String v) {
908    String url = getUrlForPackage(id);
909    if (url == null) {
910      try {
911        url = getPackageUrlFromBuildList(id);
912      } catch (Exception ignored) {
913
914      }
915    }
916    if (url == null) {
917      throw new FHIRException("Unable to resolve package id " + id + "#" + v);
918    }
919    if (url.contains("/ImplementationGuide/")) {
920      url = url.substring(0, url.indexOf("/ImplementationGuide/"));
921    }
922    String pu = Utilities.pathURL(url, "package-list.json");
923
924    PackageList pl;
925    try {
926      pl = PackageList.fromUrl(pu);
927    } catch (Exception e) {
928      String pv = Utilities.pathURL(url, v, "package.tgz");
929      try {
930        return new InputStreamWithSrc(fetchFromUrlSpecific(pv, false), pv, v);
931      } catch (Exception e1) {
932        throw new FHIRException("Error fetching package directly (" + pv + "), or fetching package list for " + id + " from " + pu + ": " + e1.getMessage(), e1);
933      }
934    }
935    if (!id.equals(pl.pid()))
936      throw new FHIRException("Package ids do not match in " + pu + ": " + id + " vs " + pl.pid());
937    for (PackageListEntry vo : pl.versions()) {
938      if (v.equals(vo.version())) {
939
940        String u = Utilities.pathURL(vo.path(), "package.tgz");
941        return new InputStreamWithSrc(fetchFromUrlSpecific(u, true), u, v);
942      }
943    }
944
945    return null;
946  }
947
948
949  // ---------- Current Build SubSystem --------------------------------------------------------------------------------------
950
951  private String fetchVersionTheOldWay(String id) throws IOException {
952    String url = getUrlForPackage(id);
953    if (url == null) {
954      try {
955        url = getPackageUrlFromBuildList(id);
956      } catch (Exception e) {
957        url = null;
958      }
959    }
960    if (url == null) {
961      throw new FHIRException("Unable to resolve package id " + id);
962    }
963    PackageList pl = PackageList.fromUrl(Utilities.pathURL(url, "package-list.json"));
964    if (!id.equals(pl.pid()))
965      throw new FHIRException("Package ids do not match in " + pl.source() + ": " + id + " vs " + pl.pid());
966    for (PackageListEntry vo : pl.versions()) {
967      if (vo.current()) {
968        return vo.version();
969      }
970    }
971
972    return null;
973  }
974
975  private String getUrlForPackage(String id) {
976    if (CommonPackages.ID_XVER.equals(id)) {
977      return "https://fhir.org/packages/hl7.fhir.xver-extensions";
978    }
979    return null;
980  }
981
982  public List<String> listPackages() {
983    List<String> res = new ArrayList<>();
984    for (File f : cacheFolder.listFiles()) {
985      if (f.isDirectory() && f.getName().contains("#")) {
986        res.add(f.getName());
987      }
988    }
989    return res;
990  }
991
992  public interface CacheLockFunction<T> {
993    T get() throws IOException;
994  }
995
996  public class BuildRecordSorter implements Comparator<BuildRecord> {
997
998    @Override
999    public int compare(BuildRecord arg0, BuildRecord arg1) {
1000      return arg1.date.compareTo(arg0.date);
1001    }
1002  }
1003
1004  public class BuildRecord {
1005
1006    private final String url;
1007    private final String packageId;
1008    private final String repo;
1009    private final Date date;
1010
1011    public BuildRecord(String url, String packageId, String repo, Date date) {
1012      super();
1013      this.url = url;
1014      this.packageId = packageId;
1015      this.repo = repo;
1016      this.date = date;
1017    }
1018
1019    public String getUrl() {
1020      return url;
1021    }
1022
1023    public String getPackageId() {
1024      return packageId;
1025    }
1026
1027    public String getRepo() {
1028      return repo;
1029    }
1030
1031    public Date getDate() {
1032      return date;
1033    }
1034
1035  }
1036
1037  public boolean packageExists(String id, String ver) throws IOException {
1038    if (packageInstalled(id, ver)) {
1039      return true;
1040    }
1041    for (PackageServer s : getPackageServers()) {
1042      if (new PackageClient(s).exists(id, ver)) {
1043        return true;
1044      }
1045    }
1046    return false;
1047  }
1048
1049  public boolean packageInstalled(String id, String version) {
1050    for (NpmPackage p : temporaryPackages) {
1051      if (p.name().equals(id) && ("current".equals(version) || "dev".equals(version) || p.version().equals(version))) {
1052        return true;
1053      }
1054      if (p.name().equals(id) && Utilities.noString(version)) {
1055        return true;
1056      }
1057    }
1058
1059    for (String f : Utilities.sorted(cacheFolder.list())) {
1060      if (f.equals(id + "#" + version) || (Utilities.noString(version) && f.startsWith(id + "#"))) {
1061        return true;
1062      }
1063    }
1064    if ("dev".equals(version))
1065      return packageInstalled(id, "current");
1066    else
1067      return false;
1068  }
1069
1070  public boolean isSuppressErrors() {
1071    return suppressErrors;
1072  }
1073
1074  public void setSuppressErrors(boolean suppressErrors) {
1075    this.suppressErrors = suppressErrors;
1076  }
1077
1078  public static IPackageProvider getPackageProvider() {
1079    return packageProvider;
1080  }
1081
1082  public static void setPackageProvider(IPackageProvider packageProvider) {
1083    FilesystemPackageCacheManager.packageProvider = packageProvider;
1084  }
1085
1086
1087}