001package org.hl7.fhir.utilities.i18n; 002 003import java.io.File; 004import java.io.FileNotFoundException; 005import java.io.IOException; 006import java.nio.file.Files; 007import java.util.ArrayList; 008import java.util.Collections; 009import java.util.Comparator; 010import java.util.HashMap; 011import java.util.HashSet; 012import java.util.List; 013import java.util.Map; 014import java.util.Set; 015 016import org.hl7.fhir.utilities.StringPair; 017import org.hl7.fhir.utilities.TextFile; 018import org.hl7.fhir.utilities.Utilities; 019import org.hl7.fhir.utilities.filesystem.DirectoryVisitor; 020import org.hl7.fhir.utilities.filesystem.DirectoryVisitor.IDirectoryVisitorImplementation; 021import org.hl7.fhir.utilities.filesystem.ManagedFileAccess; 022 023/** 024 * This class checks that all the i18n constants and declarations are consistent, 025 * and then generates / updates the .po source files, and then updates the .properties files 026 * 027 * It needs to be run whenever 028 * (a) New constants are added to the java code 029 * (b) An implementer contributes translations in a .po source file 030 * 031 * It takes 3 parameters: 032 * * path to the local copy of the core repo 033 * * path to the local copy of the ig-publisher repo 034 * * path to the local copy of the fhirserver repo 035 */ 036public class POGenerator { 037 038 public class PropertyValue extends StringPair { 039 private boolean used; 040 041 public PropertyValue(String name, String value) { 042 super(name, value); 043 // TODO Auto-generated constructor stub 044 } 045 046 public String getBaseName() { 047 String res = getName(); 048 if (res.endsWith("_one")) { 049 res = res.substring(0, res.length()-4); 050 } else if (res.endsWith("_other")) { 051 res = res.substring(0, res.length()-6); 052 } 053 return res; 054 } 055 056 } 057 058 public class ConstantDefinition { 059 private String name; 060 private String sname; 061 private boolean defined; 062 private boolean used; 063 } 064 065 public class POObjectSorter implements Comparator<POObject> { 066 067 @Override 068 public int compare(POObject o1, POObject o2) { 069 return o1.id.compareTo(o2.id); 070 } 071 } 072 073 private class POObject { 074 private boolean orphan = true; 075 private boolean duplicate; 076 private String id; 077 private String msgid; 078 private String oldMsgId; 079 private String msgidPlural; 080 private String comment; 081 private List<String> msgstr = new ArrayList<String>(); 082 } 083 084 085 public static void main(String[] args) throws IOException { 086 new POGenerator().execute(args[0], args[1], args[2]); 087 } 088 089 private List<String> prefixes = new ArrayList<>(); 090 private int noTrans = 0; 091 092 private void execute(String core, String igpub, String pascal) throws IOException { 093 String source = Utilities.path(core, "/org.hl7.fhir.utilities/src/main/resources"); 094 if (checkState(source, core, igpub, pascal)) { 095 generate(source, "rendering-phrases.properties", "rendering-phrases-en.po", null, 2); 096 generate(source, "rendering-phrases.properties", "rendering-phrases-de.po", "rendering-phrases_de.properties", 2); 097 generate(source, "rendering-phrases.properties", "rendering-phrases-es.po", "rendering-phrases_es.properties", 3); 098 generate(source, "rendering-phrases.properties", "rendering-phrases-ja.po", "rendering-phrases_ja.properties", 2); 099 generate(source, "rendering-phrases.properties", "rendering-phrases-nl.po", "rendering-phrases_nl.properties", 2); 100 generate(source, "rendering-phrases.properties", "rendering-phrases-pt_BR.po", "rendering-phrases_pt-BR.properties", 2); 101 102 generate(source, "Messages.properties", "validator-messages-en.po", null, 2); 103 generate(source, "Messages.properties", "validator-messages-de.po", "Messages_de.properties", 2); 104 generate(source, "Messages.properties", "validator-messages-es.po", "Messages_es.properties", 3); 105 generate(source, "Messages.properties", "validator-messages-ja.po", "Messages_ja.properties", 2); 106 generate(source, "Messages.properties", "validator-messages-nl.po", "Messages_nl.properties", 2); 107 generate(source, "Messages.properties", "validator-messages-pt_BR.po", "Messages_pt-BR.properties", 2); 108 109 System.out.println("Finished"); 110 } 111 } 112 113 private boolean checkState(String source, String core, String igpub, String pascal) throws IOException { 114 System.out.println("Checking..."); 115 List<PropertyValue> props = loadProperties(Utilities.path(source, "rendering-phrases.properties"), true); 116 List<ConstantDefinition> consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/RenderingI18nContext.java")); 117 boolean ok = true; 118 for (ConstantDefinition cd : consts) { 119 boolean found = false; 120 for (PropertyValue p : props) { 121 String pn = p.getBaseName(); 122 if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) { 123 found = true; 124 p.used = true; 125 } 126 } 127 cd.defined = found; 128 } 129 scanJavaSource(core, consts, "RenderingI18nContext", "RenderingContext"); 130 scanJavaSource(igpub, consts, "RenderingI18nContext", "RenderingContext"); 131 scanPascalSource(pascal, props); 132 133 Set<String> pns = new HashSet<>(); 134 for (PropertyValue p : props) { 135 if (!p.used) { 136 ok = false; 137 System.out.println("Error: PV "+p.getName()+ " provided but not used"); 138 } 139 if (!pns.contains(p.getName())) { 140 pns.add(p.getName()); 141 } else { 142 System.out.println("Error: PV "+p.getName()+ " duplicated"); 143 } 144 if (p.getValue().contains("\\n")) { 145 System.out.println("Error: PV "+p.getName()+ " has a \\n"); 146 } 147 } 148 149 for (ConstantDefinition cd : consts) { 150 if (!cd.defined && !cd.used) { 151 System.out.println("Error: "+cd.name+ " not defined or used"); 152 ok = false; 153 } else if (!cd.defined) { 154 ok = false; 155 System.out.println("Error: msg for "+cd.name+ " not found at "+cd.sname); 156 } else if (!cd.used) { 157 System.out.println("Warning: const "+cd.name+ " not used"); 158 ok = false; 159 } 160 } 161 162 props = loadProperties(Utilities.path(source, "Messages.properties"), true); 163 consts = loadConstants(Utilities.path(core, "/org.hl7.fhir.utilities/src/main/java/org/hl7/fhir/utilities/i18n/I18nConstants.java")); 164 for (ConstantDefinition cd : consts) { 165 boolean found = false; 166 for (PropertyValue p : props) { 167 String pn = p.getBaseName(); 168 if (pn.equals(cd.sname) || p.getName().equals(cd.sname)) { 169 found = true; 170 p.used = true; 171 } 172 } 173 cd.defined = found; 174 } 175 176 scanJavaSource(core, consts, "I18nConstants"); 177 scanJavaSource(igpub, consts, "I18nConstants"); 178 scanPascalSource(pascal, props); 179 180 pns = new HashSet<>(); 181 for (PropertyValue p : props) { 182 if (!p.used) { 183 ok = false; 184 System.out.println("Error: PV "+p.getName()+ " provided but not used"); 185 } 186 if (!pns.contains(p.getName())) { 187 pns.add(p.getName()); 188 } else { 189 System.out.println("Error: PV "+p.getName()+ " duplicated"); 190 } 191 if (p.getValue().contains("\\n")) { 192 System.out.println("Error: PV "+p.getName()+ " has a \\n"); 193 } 194 195 } 196 197 for (ConstantDefinition cd : consts) { 198 if (!cd.defined && !cd.used) { 199 System.out.println("Error: "+cd.name+ " not defined or used"); 200 ok = false; 201 } else if (!cd.defined) { 202 ok = false; 203 System.out.println("Error: msg for "+cd.name+ " not found @ "+cd.sname); 204 } else if (!cd.used) { 205 System.out.println("Warning: const "+cd.name+ " not used"); 206 ok = false; 207 } 208 } 209 if (ok) { 210 System.out.println("No Errors Found"); 211 } else { 212 System.out.println("Errors Found, so not continuing"); 213 } 214 215 return ok; 216 } 217 218 private class JavaScanner implements IDirectoryVisitorImplementation { 219 List<ConstantDefinition> consts; 220 List<String> names; 221 222 @Override 223 public boolean enterDirectory(File f) throws IOException { 224 return !Utilities.existsInList(f.getName(), "model", "formats"); 225 } 226 227 @Override 228 public boolean visitFile(File file) throws IOException { 229 String source = TextFile.fileToString(file); 230 for (ConstantDefinition cd : consts) { 231 if (!cd.used) { 232 boolean found = false; 233 for (String n : names) { 234 if (source.contains(n+"."+cd.name+",")) { 235 found = true; 236 } 237 if (source.contains(n+"."+cd.name+")")) { 238 found = true; 239 } 240 if (source.contains(n+"."+cd.name+" :")) { 241 found = true; 242 } 243 if (source.contains(n+"."+cd.name+";")) { 244 found = true; 245 } 246 } 247 if (found) { 248 cd.used = true; 249 } 250 } 251 } 252 return true; 253 } 254 } 255 256 private void scanJavaSource(String path, List<ConstantDefinition> consts, String... names) throws FileNotFoundException, IOException { 257 JavaScanner scanner = new JavaScanner(); 258 scanner.consts = consts; 259 scanner.names = new ArrayList<String>(); 260 for (String s : names) { 261 scanner.names.add(s); 262 } 263 DirectoryVisitor.visitDirectory(scanner, path, "java"); 264 } 265 266 private class PascalScanner implements IDirectoryVisitorImplementation { 267 private List<PropertyValue> defs; 268 269 @Override 270 public boolean enterDirectory(File directory) throws IOException { 271 return true; 272 } 273 274 @Override 275 public boolean visitFile(File file) throws IOException { 276 String source = TextFile.fileToString(file); 277 for (PropertyValue pv : defs) { 278 if (!pv.used) { 279 boolean found = false; 280 String pn = pv.getBaseName(); 281 if (source.contains("'"+pn+"'")) { 282 found = true; 283 } 284 if (found) { 285 pv.used = true; 286 } 287 } 288 } 289 return true; 290 } 291 } 292 293 private void scanPascalSource(String path, List<PropertyValue> defs) throws FileNotFoundException, IOException { 294 PascalScanner scanner = new PascalScanner(); 295 scanner.defs = defs; 296 DirectoryVisitor.visitDirectory(scanner, path, "pas"); 297 } 298 299 300 private List<ConstantDefinition> loadConstants(String path) throws FileNotFoundException, IOException { 301 List<ConstantDefinition> res = new ArrayList<POGenerator.ConstantDefinition>(); 302 for (String line : TextFile.fileToLines(path)) { 303 if (line.contains("public static final String") && !line.trim().startsWith("//")) { 304 int i = line.indexOf("public static final String") + "public static final String".length(); 305 String[] p = line.substring(i).split("\\="); 306 if (p.length == 2) { 307 String n = p[0].trim(); 308 String v = p[1].trim().replace("\"", "").replace(";", ""); 309 ConstantDefinition cd = new ConstantDefinition(); 310 cd.name = n; 311 cd.sname = v; 312 res.add(cd); 313 } 314 } 315 } 316 return res; 317 } 318 319 private void generate(String source, String src, String dest, String tgt, int count) throws IOException { 320 // load the destination file 321 // load the source file 322 // update the destination object set for changes from the source file 323 // save the destination file 324 List<POObject> objects = loadPOFile(Utilities.path(source, "source", dest)); 325 List<PropertyValue> props = loadProperties(Utilities.path(source, src), false); 326 for (PropertyValue e : props) { 327 String name = e.getName(); 328 int mode = 0; 329 if (name.endsWith("_one")) { 330 mode = 1; 331 name = name.substring(0, name.length() - 4); 332 } else if (name.endsWith("_other")) { 333 mode = 2; 334 name = name.substring(0, name.length() - 6); 335 } 336 337 POObject o = findObject(objects, name); 338 if (o == null) { 339 if (mode > 1) { 340 throw new Error("Not right"); 341 } 342 o = new POObject(); 343 o.id = name; 344 o.comment = name; 345 objects.add(o); 346 o.msgid = e.getValue(); 347 o.orphan = false; 348 } else { 349 update(o, mode, e.getValue()); 350 } 351 } 352 objects.removeIf(o -> o.orphan); 353 Collections.sort(objects, new POObjectSorter()); 354 Map<String, Integer> sources = new HashMap<>(); 355 Set<String> dups = new HashSet<>(); 356 for (POObject o : objects) { 357 if (sources.containsKey(o.msgid)) { 358 Integer c = sources.get(o.msgid)+1; 359 sources.put(o.msgid, c); 360 dups.add(o.msgid); 361// System.out.println("Duplicate in "+dest.substring(dest.lastIndexOf("/")+1)+": "+o.msgid+" on ("+CommaSeparatedStringBuilder.join(",", listIds(objects, o.msgid))+")"); 362 } else { 363 sources.put(o.msgid, 1); 364 } 365 } 366 for (POObject o : objects) { 367 Integer c = sources.get(o.msgid); 368 if (c > 1) { 369 o.duplicate = true; 370 } 371 } 372 savePOFile(Utilities.path(source, "source", dest), objects, count, false); 373 if (tgt == null) { 374 savePOFile(Utilities.path(source, "source", "transifex", dest), objects, count, true); 375 } else { 376 savePOFile(Utilities.path(source, "source", "transifex", "translations", dest), objects, count, true); 377 } 378 if (tgt != null) { 379 savePropFile(Utilities.path(source, tgt), objects); 380 } 381 } 382 383 private Set<String> listIds(List<POObject> objects, String msgid) { 384 Set<String> res = new HashSet<>(); 385 for (POObject o : objects) { 386 if (o.msgid.equals(msgid)) { 387 res.add(o.id); 388 } 389 } 390 return res; 391 } 392 393 private void savePOFile(String dest, List<POObject> objects, int count, boolean tfxMode) throws IOException { 394 prefixes.clear(); 395 396 StringBuilder b = new StringBuilder(); 397 for (String p : prefixes) { 398 b.append(p); 399 b.append("\r\n"); 400 } 401 b.append("\r\n"); 402 for (POObject o : objects) { 403 // for POEdit 404 if (o.oldMsgId != null) { 405 b.append("# "+o.comment+" (!!when last translated was: "+o.oldMsgId+")\r\n"); 406 } else { 407 b.append("# "+o.comment+"\r\n"); 408 } 409 b.append("#: "+o.id+"\r\n"); 410 if (o.duplicate) { 411 b.append("msgctxt \""+o.id+"\"\r\n"); 412 } 413 String m = tfxMode && Utilities.noString(o.msgid) ? "-- no content: do not translate #"+(++noTrans )+" --" : o.msgid; 414 b.append("msgid \""+wrapQuotes(m)+"\"\r\n"); 415 if (o.msgidPlural != null) { 416 b.append("msgid_plural \""+wrapQuotes(o.msgidPlural)+"\"\r\n"); 417 while (o.msgstr.size() < count) { 418 o.msgstr.add(""); 419 } 420 for (int i = 0; i < o.msgstr.size(); i++) { 421 String s = o.msgstr.get(i); 422// if (tfxMode && Utilities.noString(s)) { 423// s = Utilities.noString(i == 0 ? o.msgid : o.msgidPlural) ? "-- no content: do not translate --" : ""; 424// } 425 b.append("msgstr["+i+"] \""+wrapQuotes(s)+"\"\r\n"); 426 } 427 } else { 428 if (o.msgstr.size() == 0) { 429 b.append("msgstr \"\"\r\n"); 430 } else { 431 b.append("msgstr \""+wrapQuotes(o.msgstr.get(0))+"\"\r\n"); 432 } 433 } 434 b.append("\r\n"); 435 } 436 TextFile.stringToFile(b.toString(), dest); 437 } 438 439 private String wrapQuotes(String s) { 440 return s.replace("\"", "\\\""); 441 } 442 443 private void update(POObject o, int mode, String value) { 444 o.orphan = false; 445 if (o.comment == null) { 446 o.comment = o.id; 447 } 448 if (mode == 0) { 449 if (!value.equals(o.msgid)) { 450 // the english string has changed, and the other language string is now out of date 451 if (o.oldMsgId != null && !o.msgstr.isEmpty()) { 452 o.oldMsgId = o.msgid; 453 } 454 o.msgid = value; 455 for (int i = 0; i < o.msgstr.size(); i++) { 456 if (!Utilities.noString(o.msgstr.get(i))) { 457 o.msgstr.set(i, "!!"+o.msgstr.get(i)); 458 } 459 } 460 } else { 461 o.oldMsgId = null; 462 } 463 } else if (mode == 1) { 464 if (!value.equals(o.msgid)) { 465 // the english string has changed, and the other language string is now out of date 466 if (o.oldMsgId != null && !o.msgstr.isEmpty()) { 467 o.oldMsgId = o.msgid; 468 } 469 o.msgid = value; 470 if (o.msgstr.size() > 0 && !Utilities.noString(o.msgstr.get(0))) { 471 o.msgstr.set(0, "!!"+o.msgstr.get(0)); 472 } 473 } else { 474 o.oldMsgId = null; 475 } 476 } else if (mode == 2) { 477 if (!value.equals(o.msgidPlural)) { 478 // the english string has changed, and the other language string is now out of date 479// if (o.oldMsgId != null) { 480// o.oldMsgId = o.msgid; 481// } 482 o.msgidPlural = value; 483 if (o.msgstr.size() > 1 && !Utilities.noString(o.msgstr.get(1))) { 484 o.msgstr.set(1, "!!"+o.msgstr.get(1)); 485 } 486 } else { 487 o.oldMsgId = null; 488 } 489 } 490 } 491 492 private POObject findObject(List<POObject> objects, String name) { 493 for (POObject t : objects) { 494 if (t.id != null && t.id.equals(name)) { 495 return t; 496 } 497 } 498 return null; 499 } 500 501 private List<POObject> loadPOFile(String dest) throws FileNotFoundException, IOException { 502 List<POObject> list = new ArrayList<POGenerator.POObject>(); 503 POObject obj = null; 504 for (String line : TextFile.fileToLines(dest)) { 505 if (Utilities.noString(line)) { 506 // else 507 } else if (line.startsWith("#:")) { 508 if (obj == null || obj.id != null) { 509 obj = new POObject(); 510 list.add(obj); 511 } 512 obj.id = line.substring(2).trim(); 513 } else if (line.startsWith("# ")) { 514 if (obj == null || obj.comment != null) { 515 obj = new POObject(); 516 list.add(obj); 517 } 518 obj.comment = line.substring(1).trim(); 519 if (obj.comment.contains("!!when")) { 520 obj.oldMsgId = obj.comment.substring(obj.comment.indexOf("!!when")); 521 obj.comment = obj.comment.substring(0, obj.comment.indexOf("!!when")-1); 522 obj.oldMsgId = obj.oldMsgId.substring(obj.oldMsgId.indexOf(": ")+2).trim(); 523 obj.oldMsgId = obj.oldMsgId.substring(0, obj.oldMsgId.length()-1); 524 } 525 } else if (obj == null) { 526 prefixes.add(line); 527 } else if (line.startsWith("#|")) { 528 // retired use of #| because it caused problems with the tools 529 obj.oldMsgId = line.substring(2).trim(); 530 if (obj.oldMsgId.startsWith("msgid ")) { 531 obj.oldMsgId = trimQuotes(obj.oldMsgId.substring(6)); 532 } 533 } else if (line.startsWith("msgid ")) { 534 obj.msgid = trimQuotes(line.substring(5).trim()); 535 if (obj.msgid.endsWith("("+obj.id+")")) { 536 obj.msgid = obj.msgid.substring(0, obj.msgid.length() - (obj.id.length()+3)); 537 } 538 } else if (line.startsWith("msgid_plural ")) { 539 obj.msgidPlural = trimQuotes(line.substring(12).trim()); 540 } else if (line.startsWith("msgstr ")) { 541 obj.msgstr.add(trimQuotes(line.substring(6).trim())); 542 } else if (line.startsWith("msgctxt ")) { 543 // ignore this 544 } else if (line.startsWith("msgstr[")) { 545 String s = line.substring(7); 546 int i = s.indexOf("]"); 547 int c = Integer.valueOf(s.substring(0, i)); 548 s = trimQuotes(s.substring(i+1).trim()); 549 while (s.startsWith("!!")) { 550 s = s.substring(2); 551 } 552 if (c != obj.msgstr.size()) { 553 System.out.println("index issue"); 554 } else { // if (!Utilities.noString(s)) { 555 obj.msgstr.add(s); 556 } 557 } else { 558 System.out.println("unknown line: "+line); 559 } 560 } 561 return list; 562 } 563 564 private String trimQuotes(String s) { 565 s = s.trim(); 566 if (s.startsWith("\"")) { 567 s = s.substring(1); 568 } 569 if (s.endsWith("\"")) { 570 s = s.substring(0, s.length()-1); 571 } 572 return s.trim().replace("\\\"", "\""); 573 } 574 575 private List<PropertyValue> loadProperties(String source, boolean checking) throws IOException { 576 List<PropertyValue> res = new ArrayList<>(); 577 File src = ManagedFileAccess.file(source); 578 List<String> lines = Files.readAllLines(src.toPath()); 579 for (String line : lines) { 580 if (!line.startsWith("#") && line.contains("=")) { 581 String n = line.substring(0, line.indexOf("=")).trim(); 582 String v = line.substring(line.indexOf("=")+1).trim(); 583 if (checking || !(v.length() == 3 && v.startsWith("{") && v.endsWith("}"))) { 584 res.add(new PropertyValue(n, v)); 585 } 586 } 587 } 588 return res; 589 } 590 591 private void savePropFile(String tgt, List<POObject> objects) throws IOException { 592 String nameLine = TextFile.fileToLines(tgt)[0]; 593 String[] parts = nameLine.substring(1).trim().split("\\="); 594 String[] names = parts[1].split("\\,"); 595 596 StringBuilder b = new StringBuilder(); 597 b.append(nameLine+"\r\n"); 598 for (POObject o : objects) { 599 if (o.msgidPlural == null) { 600 String v= o.msgstr.size() > 0 ? o.msgstr.get(0) : ""; 601 if (!Utilities.noString(v)) { 602 b.append(o.id+" = "+v+"\r\n"); 603 } 604 } else { 605 for (int i = 0; i < names.length; i++) { 606 String v = (o.msgstr.size() > i ? o.msgstr.get(i) : ""); 607 if (!Utilities.noString(v)) { 608 b.append(o.id+"_"+names[i].trim()+" = "+v+"\r\n"); 609 } 610 } 611 } 612 } 613 614 TextFile.stringToFile(b.toString(), tgt); 615 } 616 617 618}