001package org.hl7.fhir.utilities.i18n;
002
003import java.io.File;
004import java.io.FileNotFoundException;
005import java.io.IOException;
006import java.nio.file.Files;
007import java.util.ArrayList;
008import java.util.Collections;
009import java.util.Comparator;
010import java.util.HashMap;
011import java.util.HashSet;
012import java.util.List;
013import java.util.Map;
014import java.util.Set;
015
016import org.hl7.fhir.utilities.StringPair;
017import org.hl7.fhir.utilities.TextFile;
018import org.hl7.fhir.utilities.Utilities;
019import org.hl7.fhir.utilities.filesystem.DirectoryVisitor;
020import org.hl7.fhir.utilities.filesystem.DirectoryVisitor.IDirectoryVisitorImplementation;
021import org.hl7.fhir.utilities.filesystem.ManagedFileAccess;
022
023/**
024 * This class checks that all the i18n constants and declarations are consistent,
025 * and then generates / updates the .po source files, and then updates the .properties files
026 * 
027 * It needs to be run whenever
028 *   (a) New constants are added to the java code
029 *   (b) An implementer contributes translations in a .po source file
030 *   
031 * It takes 3 parameters:
032 *   * path to the local copy of the core repo
033 *   * path to the local copy of the ig-publisher repo
034 *   * path to the local copy of the fhirserver repo
035 */
036public class POGenerator {
037
038  public class PropertyValue extends StringPair {
039    private boolean used;
040    
041    public PropertyValue(String name, String value) {
042      super(name, value);
043      // TODO Auto-generated constructor stub
044    }
045
046    public String getBaseName() {
047      String res = getName();
048      if (res.endsWith("_one")) {
049        res = res.substring(0, res.length()-4);
050      } else if (res.endsWith("_other")) {
051        res = res.substring(0, res.length()-6);
052      }
053      return res;
054    }
055
056  }
057
058  public class ConstantDefinition {
059    private String name;
060    private String sname;
061    private boolean defined;
062    private boolean used;
063  }
064
065  public class POObjectSorter implements Comparator<POObject> {
066
067    @Override
068    public int compare(POObject o1, POObject o2) {
069      return o1.id.compareTo(o2.id);
070    }
071  }
072
073  private class POObject {
074    private boolean orphan = true;
075    private boolean duplicate;
076    private String id;
077    private String msgid;
078    private String oldMsgId;
079    private String msgidPlural;
080    private String comment;
081    private List<String> msgstr  = new ArrayList<String>();
082  }
083  
084
085  public static void main(String[] args) throws IOException {
086    new POGenerator().execute(args[0], args[1], args[2]);
087  }
088
089  private List<String> prefixes = new ArrayList<>();
090  private int noTrans = 0;
091
092  private void execute(String core, String igpub, String pascal) throws IOException {
093    String source = Utilities.path(core, "/org.hl7.fhir.utilities/src/main/resources");
094    if (checkState(source, core, igpub, pascal)) {
095      generate(source, "rendering-phrases.properties",  "rendering-phrases-en.po",       null, 2);
096      generate(source, "rendering-phrases.properties",  "rendering-phrases-de.po",    "rendering-phrases_de.properties", 2);
097      generate(source, "rendering-phrases.properties",  "rendering-phrases-es.po",    "rendering-phrases_es.properties", 3);
098      generate(source, "rendering-phrases.properties",  "rendering-phrases-ja.po",    "rendering-phrases_ja.properties", 2);
099      generate(source, "rendering-phrases.properties",  "rendering-phrases-nl.po",    "rendering-phrases_nl.properties", 2);
100      generate(source, "rendering-phrases.properties",  "rendering-phrases-pt_BR.po", "rendering-phrases_pt-BR.properties", 2);
101
102      generate(source, "Messages.properties", "validator-messages-en.po",    null, 2);
103      generate(source, "Messages.properties", "validator-messages-de.po",    "Messages_de.properties", 2);
104      generate(source, "Messages.properties", "validator-messages-es.po",    "Messages_es.properties", 3);
105      generate(source, "Messages.properties", "validator-messages-ja.po",    "Messages_ja.properties", 2);
106      generate(source, "Messages.properties", "validator-messages-nl.po",    "Messages_nl.properties", 2);
107      generate(source, "Messages.properties", "validator-messages-pt_BR.po", "Messages_pt-BR.properties", 2);
108
109      System.out.println("Finished");
110    } 
111  }
112
113  private boolean checkState(String source, String core, String igpub, String pascal) throws IOException {
114    System.out.println("Checking...");
115    List<PropertyValue> props = loadProperties(Utilities.path(source, "rendering-phrases.properties"), true);
116    List<ConstantDefinition> consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/RenderingI18nContext.java"));
117    boolean ok = true;
118    for (ConstantDefinition cd : consts) {
119      boolean found = false;
120      for (PropertyValue p : props) {
121        String pn = p.getBaseName();        
122        if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) {
123          found = true;
124          p.used = true;
125        }
126      }
127      cd.defined = found;
128    }
129    scanJavaSource(core, consts, "RenderingI18nContext", "RenderingContext");
130    scanJavaSource(igpub, consts, "RenderingI18nContext", "RenderingContext");
131    scanPascalSource(pascal, props);
132    
133    Set<String> pns = new HashSet<>();
134    for (PropertyValue p : props) {
135      if (!p.used) {
136        ok = false;
137        System.out.println("Error: PV "+p.getName()+ " provided but not used");   
138      }
139      if (!pns.contains(p.getName())) {
140        pns.add(p.getName());
141      } else {
142        System.out.println("Error: PV "+p.getName()+ " duplicated");
143      }
144      if (p.getValue().contains("\\n")) {
145        System.out.println("Error: PV "+p.getName()+ " has a \\n");
146      }
147    }
148    
149    for (ConstantDefinition cd : consts) {
150      if (!cd.defined && !cd.used) {
151        System.out.println("Error: "+cd.name+ " not defined or used");        
152        ok = false;
153      } else if (!cd.defined) {
154        ok = false;
155        System.out.println("Error: msg for "+cd.name+ " not found at "+cd.sname);
156      } else if (!cd.used) {
157        System.out.println("Warning: const "+cd.name+ " not used");
158        ok = false;
159      }
160    }
161
162    props = loadProperties(Utilities.path(source, "Messages.properties"), true);
163    consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/I18nConstants.java"));
164    for (ConstantDefinition cd : consts) {
165      boolean found = false;
166      for (PropertyValue p : props) {
167        String pn = p.getBaseName();
168        if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) {
169          found = true;
170          p.used = true;
171        }
172      }
173      cd.defined = found;
174    }
175
176    scanJavaSource(core, consts, "I18nConstants");
177    scanJavaSource(igpub, consts, "I18nConstants");
178    scanPascalSource(pascal, props);
179
180    pns = new HashSet<>();
181    for (PropertyValue p : props) {
182      if (!p.used) {
183        ok = false;
184        System.out.println("Error: PV "+p.getName()+ " provided but not used");   
185      }
186      if (!pns.contains(p.getName())) {
187        pns.add(p.getName());
188      } else {
189        System.out.println("Error: PV "+p.getName()+ " duplicated");
190      }      
191      if (p.getValue().contains("\\n")) {
192        System.out.println("Error: PV "+p.getName()+ " has a \\n");
193      }
194
195    }
196    
197    for (ConstantDefinition cd : consts) {
198      if (!cd.defined && !cd.used) {
199        System.out.println("Error: "+cd.name+ " not defined or used");        
200        ok = false;
201      } else if (!cd.defined) {
202        ok = false;
203        System.out.println("Error: msg for "+cd.name+ " not found @ "+cd.sname);
204      } else if (!cd.used) {
205        System.out.println("Warning: const "+cd.name+ " not used");
206        ok = false;
207      }
208    }
209    if (ok) {
210      System.out.println("No Errors Found");
211    } else {
212      System.out.println("Errors Found, so not continuing");  
213    }
214
215    return ok;
216  }
217
218  private class JavaScanner implements IDirectoryVisitorImplementation {
219    List<ConstantDefinition> consts;
220    List<String> names;
221    
222    @Override
223    public boolean enterDirectory(File f) throws IOException {
224      return !Utilities.existsInList(f.getName(), "model", "formats");
225    }
226
227    @Override
228    public boolean visitFile(File file) throws IOException {
229      String source = TextFile.fileToString(file);
230      for (ConstantDefinition cd : consts) {
231        if (!cd.used) {
232          boolean found = false;
233          for (String n : names) {
234            if (source.contains(n+"."+cd.name+",")) {
235              found = true;
236            } 
237            if (source.contains(n+"."+cd.name+")")) {
238              found = true;
239            } 
240            if (source.contains(n+"."+cd.name+" :")) {
241              found = true;
242            } 
243            if (source.contains(n+"."+cd.name+";")) {
244              found = true;
245            } 
246          } 
247          if (found) {
248            cd.used = true;
249          }
250        }  
251      }
252      return true;
253    }
254  }
255  
256  private void scanJavaSource(String path, List<ConstantDefinition> consts, String... names) throws FileNotFoundException, IOException {
257    JavaScanner scanner = new JavaScanner();
258    scanner.consts = consts;
259    scanner.names = new ArrayList<String>();
260    for (String s : names) {
261      scanner.names.add(s);
262    }
263    DirectoryVisitor.visitDirectory(scanner, path, "java");
264  }
265
266  private class PascalScanner implements IDirectoryVisitorImplementation {
267    private List<PropertyValue> defs;
268    
269    @Override
270    public boolean enterDirectory(File directory) throws IOException {
271      return true;
272    }
273
274    @Override
275    public boolean visitFile(File file) throws IOException {
276      String source = TextFile.fileToString(file);
277      for (PropertyValue pv : defs) {
278        if (!pv.used) {
279          boolean found = false;
280          String pn = pv.getBaseName();
281          if (source.contains("'"+pn+"'")) {
282            found = true;
283          } 
284          if (found) {
285            pv.used = true;
286          }
287        }
288      }
289      return true;
290    }
291  }
292  
293  private void scanPascalSource(String path, List<PropertyValue> defs) throws FileNotFoundException, IOException {
294    PascalScanner scanner = new PascalScanner();
295    scanner.defs = defs;
296    DirectoryVisitor.visitDirectory(scanner, path, "pas");
297  }
298
299  
300  private List<ConstantDefinition> loadConstants(String path) throws FileNotFoundException, IOException {
301    List<ConstantDefinition> res = new ArrayList<POGenerator.ConstantDefinition>();
302    for (String line : TextFile.fileToLines(path)) {
303      if (line.contains("public static final String") && !line.trim().startsWith("//")) {
304        int i = line.indexOf("public static final String") + "public static final String".length();
305        String[] p = line.substring(i).split("\\=");
306        if (p.length == 2) {
307          String n = p[0].trim();
308          String v = p[1].trim().replace("\"", "").replace(";", "");
309          ConstantDefinition cd = new ConstantDefinition();
310          cd.name = n;
311          cd.sname = v;
312          res.add(cd);
313        }
314      }
315    }
316    return res;
317  }
318
319  private void generate(String source, String src, String dest, String tgt, int count) throws IOException {
320    // load the destination file 
321    // load the source file 
322    // update the destination object set for changes from the source file
323    // save the destination file 
324    List<POObject> objects = loadPOFile(Utilities.path(source, "source", dest));
325    List<PropertyValue> props = loadProperties(Utilities.path(source, src), false);
326    for (PropertyValue e : props) {
327      String name = e.getName();
328      int mode = 0;
329      if (name.endsWith("_one")) {
330        mode = 1;
331        name = name.substring(0, name.length() - 4);
332      } else if (name.endsWith("_other")) {
333        mode = 2;
334        name = name.substring(0, name.length() - 6);
335      } 
336
337      POObject o = findObject(objects, name);
338      if (o == null) {
339        if (mode > 1) {
340          throw new Error("Not right");
341        }
342        o = new POObject();
343        o.id = name;
344        o.comment = name;
345        objects.add(o);
346        o.msgid = e.getValue();
347        o.orphan = false;
348      } else {
349        update(o, mode, e.getValue());
350      }
351    }
352    objects.removeIf(o -> o.orphan);
353    Collections.sort(objects, new POObjectSorter());
354    Map<String, Integer> sources = new HashMap<>();
355    Set<String> dups = new HashSet<>();
356    for (POObject o : objects) {
357      if (sources.containsKey(o.msgid)) {
358        Integer c = sources.get(o.msgid)+1;
359        sources.put(o.msgid, c);
360        dups.add(o.msgid);
361//        System.out.println("Duplicate in "+dest.substring(dest.lastIndexOf("/")+1)+": "+o.msgid+" on ("+CommaSeparatedStringBuilder.join(",", listIds(objects, o.msgid))+")");
362      } else {
363        sources.put(o.msgid, 1);
364      }
365    }
366    for (POObject o : objects) {
367      Integer c = sources.get(o.msgid);
368      if (c > 1) {
369        o.duplicate = true;
370      }
371    }
372    savePOFile(Utilities.path(source, "source", dest), objects, count, false);
373    if (tgt == null) {
374      savePOFile(Utilities.path(source, "source", "transifex", dest), objects, count, true);
375    } else {
376      savePOFile(Utilities.path(source, "source", "transifex", "translations", dest), objects, count, true);
377    }
378    if (tgt != null) {
379      savePropFile(Utilities.path(source, tgt), objects);
380    }
381  }
382
383  private Set<String> listIds(List<POObject> objects, String msgid) {
384    Set<String> res = new HashSet<>();
385    for (POObject o : objects) {
386      if (o.msgid.equals(msgid)) {
387        res.add(o.id);
388      }
389    }
390    return res;
391  }
392
393  private void savePOFile(String dest, List<POObject> objects, int count, boolean tfxMode) throws IOException {
394    prefixes.clear();
395    
396    StringBuilder b = new StringBuilder();
397    for (String p : prefixes) {
398      b.append(p);
399      b.append("\r\n");
400    }
401    b.append("\r\n");
402    for (POObject o : objects) {
403      // for POEdit
404      if (o.oldMsgId != null) {
405        b.append("# "+o.comment+" (!!when last translated was: "+o.oldMsgId+")\r\n");        
406      } else {
407        b.append("# "+o.comment+"\r\n");
408      }
409      b.append("#: "+o.id+"\r\n");
410      if (o.duplicate) {
411        b.append("msgctxt \""+o.id+"\"\r\n");        
412      } 
413      String m = tfxMode && Utilities.noString(o.msgid) ? "-- no content: do not translate #"+(++noTrans )+" --" : o.msgid; 
414      b.append("msgid \""+wrapQuotes(m)+"\"\r\n");
415      if (o.msgidPlural != null) {
416        b.append("msgid_plural \""+wrapQuotes(o.msgidPlural)+"\"\r\n"); 
417        while (o.msgstr.size() < count) {
418          o.msgstr.add("");
419        }
420        for (int i = 0; i < o.msgstr.size(); i++) {
421          String s = o.msgstr.get(i);
422//          if (tfxMode && Utilities.noString(s)) {
423//            s = Utilities.noString(i == 0 ? o.msgid : o.msgidPlural) ? "-- no content: do not translate --" : "";
424//          }
425          b.append("msgstr["+i+"] \""+wrapQuotes(s)+"\"\r\n");
426        }
427      } else {
428        if (o.msgstr.size() == 0) {
429          b.append("msgstr \"\"\r\n");                  
430        } else {
431          b.append("msgstr \""+wrapQuotes(o.msgstr.get(0))+"\"\r\n");                            
432        }
433      } 
434      b.append("\r\n");
435    }
436    TextFile.stringToFile(b.toString(), dest);
437  }
438
439  private String wrapQuotes(String s) {
440    return s.replace("\"", "\\\"");
441  }
442
443  private void update(POObject o, int mode, String value) {
444    o.orphan = false;
445    if (o.comment == null) {
446      o.comment = o.id;
447    }
448    if (mode == 0) {
449      if (!value.equals(o.msgid)) {
450        // the english string has changed, and the other language string is now out of date
451        if (o.oldMsgId != null && !o.msgstr.isEmpty()) {
452          o.oldMsgId = o.msgid;
453        }
454        o.msgid = value;
455        for (int i = 0; i < o.msgstr.size(); i++) {
456          if (!Utilities.noString(o.msgstr.get(i))) {
457            o.msgstr.set(i, "!!"+o.msgstr.get(i));
458          }
459        }
460      } else {
461        o.oldMsgId = null;
462      }
463    } else if (mode == 1) {
464      if (!value.equals(o.msgid)) {
465        // the english string has changed, and the other language string is now out of date 
466        if (o.oldMsgId != null && !o.msgstr.isEmpty()) {
467          o.oldMsgId = o.msgid;
468        }
469        o.msgid = value;
470        if (o.msgstr.size() > 0 && !Utilities.noString(o.msgstr.get(0))) {
471          o.msgstr.set(0, "!!"+o.msgstr.get(0));
472        }
473      } else {
474        o.oldMsgId = null;
475      }
476    } else if (mode == 2) {
477      if (!value.equals(o.msgidPlural)) {
478        // the english string has changed, and the other language string is now out of date 
479//        if (o.oldMsgId != null) {
480//          o.oldMsgId = o.msgid;
481//        }
482        o.msgidPlural = value;
483        if (o.msgstr.size() > 1 && !Utilities.noString(o.msgstr.get(1))) {
484          o.msgstr.set(1, "!!"+o.msgstr.get(1));
485        }
486      } else {
487        o.oldMsgId = null;
488      }
489    }
490  }
491
492  private POObject findObject(List<POObject> objects, String name) {
493    for (POObject t : objects) {
494      if (t.id != null && t.id.equals(name)) {
495        return t;
496      }
497    }
498    return null;
499  }
500
501  private List<POObject> loadPOFile(String dest) throws FileNotFoundException, IOException {
502    List<POObject> list = new ArrayList<POGenerator.POObject>();
503    POObject obj = null;
504    for (String line : TextFile.fileToLines(dest)) {
505      if (Utilities.noString(line)) {
506        // else 
507      } else if (line.startsWith("#:")) {
508        if (obj == null || obj.id != null) {
509          obj = new POObject();
510          list.add(obj);
511        }
512        obj.id = line.substring(2).trim();  
513      } else if (line.startsWith("# ")) {
514        if (obj == null || obj.comment != null) {
515          obj = new POObject();
516          list.add(obj);
517        }
518        obj.comment = line.substring(1).trim();
519        if (obj.comment.contains("!!when")) {
520          obj.oldMsgId = obj.comment.substring(obj.comment.indexOf("!!when"));
521          obj.comment = obj.comment.substring(0, obj.comment.indexOf("!!when")-1);
522          obj.oldMsgId = obj.oldMsgId.substring(obj.oldMsgId.indexOf(": ")+2).trim();
523          obj.oldMsgId = obj.oldMsgId.substring(0, obj.oldMsgId.length()-1);
524        }
525      } else if (obj == null) {
526        prefixes.add(line);  
527      } else if (line.startsWith("#|")) {
528        // retired use of #| because it caused problems with the tools
529        obj.oldMsgId = line.substring(2).trim();
530        if (obj.oldMsgId.startsWith("msgid ")) {
531          obj.oldMsgId = trimQuotes(obj.oldMsgId.substring(6));
532        }
533      } else if (line.startsWith("msgid ")) {
534        obj.msgid = trimQuotes(line.substring(5).trim());
535        if (obj.msgid.endsWith("("+obj.id+")")) {
536          obj.msgid = obj.msgid.substring(0, obj.msgid.length() - (obj.id.length()+3));
537        }
538      } else if (line.startsWith("msgid_plural ")) {
539        obj.msgidPlural = trimQuotes(line.substring(12).trim());
540      } else if (line.startsWith("msgstr ")) {
541        obj.msgstr.add(trimQuotes(line.substring(6).trim()));
542      } else if (line.startsWith("msgctxt ")) {
543        // ignore this
544      } else if (line.startsWith("msgstr[")) {
545        String s = line.substring(7);
546        int i = s.indexOf("]");
547        int c = Integer.valueOf(s.substring(0, i));
548        s = trimQuotes(s.substring(i+1).trim());
549        while (s.startsWith("!!")) {
550          s = s.substring(2);
551        }
552        if (c != obj.msgstr.size()) {
553          System.out.println("index issue");   
554        } else { // if (!Utilities.noString(s)) {
555          obj.msgstr.add(s);          
556        }
557      } else {
558        System.out.println("unknown line: "+line);          
559      }
560    }
561    return list;
562  }
563
564  private String trimQuotes(String s) {
565    s = s.trim();
566    if (s.startsWith("\"")) {
567      s = s.substring(1);
568    }
569    if (s.endsWith("\"")) {
570      s = s.substring(0, s.length()-1);
571    }
572    return s.trim().replace("\\\"", "\"");
573  }
574
575  private List<PropertyValue> loadProperties(String source, boolean checking) throws IOException {
576    List<PropertyValue> res = new ArrayList<>();
577    File src = ManagedFileAccess.file(source);
578    List<String> lines = Files.readAllLines(src.toPath());
579    for (String line : lines) {
580      if (!line.startsWith("#") && line.contains("=")) {
581        String n = line.substring(0, line.indexOf("=")).trim();
582        String v = line.substring(line.indexOf("=")+1).trim();
583        if (checking || !(v.length() == 3 && v.startsWith("{") && v.endsWith("}"))) {
584          res.add(new PropertyValue(n, v));
585        }
586      } 
587    }
588    return res;
589  }
590
591  private void savePropFile(String tgt, List<POObject> objects) throws IOException {
592    String nameLine = TextFile.fileToLines(tgt)[0];
593    String[] parts = nameLine.substring(1).trim().split("\\=");
594    String[] names = parts[1].split("\\,");
595    
596    StringBuilder b = new StringBuilder();
597    b.append(nameLine+"\r\n");
598    for (POObject o : objects) {
599      if (o.msgidPlural == null) {
600        String v= o.msgstr.size() > 0 ? o.msgstr.get(0) : "";
601        if (!Utilities.noString(v)) {
602          b.append(o.id+" = "+v+"\r\n");
603        }
604      } else {
605        for (int i = 0; i < names.length; i++) {
606          String v = (o.msgstr.size() > i ? o.msgstr.get(i) : "");
607          if (!Utilities.noString(v)) {
608            b.append(o.id+"_"+names[i].trim()+" = "+v+"\r\n");
609          }
610        }
611      }
612    }
613    
614    TextFile.stringToFile(b.toString(), tgt);
615  }
616
617  
618}