001 /*
002 * Copyright 2008-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005 /*
006 * Copyright (C) 2008-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021 package com.unboundid.ldap.sdk.examples;
022
023
024
025 import java.io.File;
026 import java.io.FileInputStream;
027 import java.io.InputStream;
028 import java.io.IOException;
029 import java.io.OutputStream;
030 import java.util.ArrayList;
031 import java.util.Iterator;
032 import java.util.TreeMap;
033 import java.util.LinkedHashMap;
034 import java.util.List;
035 import java.util.concurrent.atomic.AtomicLong;
036 import java.util.zip.GZIPInputStream;
037
038 import com.unboundid.ldap.sdk.Entry;
039 import com.unboundid.ldap.sdk.LDAPConnection;
040 import com.unboundid.ldap.sdk.LDAPException;
041 import com.unboundid.ldap.sdk.ResultCode;
042 import com.unboundid.ldap.sdk.Version;
043 import com.unboundid.ldap.sdk.schema.Schema;
044 import com.unboundid.ldap.sdk.schema.EntryValidator;
045 import com.unboundid.ldif.DuplicateValueBehavior;
046 import com.unboundid.ldif.LDIFException;
047 import com.unboundid.ldif.LDIFReader;
048 import com.unboundid.ldif.LDIFReaderEntryTranslator;
049 import com.unboundid.ldif.LDIFWriter;
050 import com.unboundid.util.LDAPCommandLineTool;
051 import com.unboundid.util.ThreadSafety;
052 import com.unboundid.util.ThreadSafetyLevel;
053 import com.unboundid.util.args.ArgumentException;
054 import com.unboundid.util.args.ArgumentParser;
055 import com.unboundid.util.args.BooleanArgument;
056 import com.unboundid.util.args.FileArgument;
057 import com.unboundid.util.args.IntegerArgument;
058 import com.unboundid.util.args.StringArgument;
059
060 import static com.unboundid.util.StaticUtils.*;
061
062
063
064 /**
065 * This class provides a simple tool that can be used to validate that the
066 * contents of an LDIF file are valid. This includes ensuring that the contents
067 * can be parsed as valid LDIF, and it can also ensure that the LDIF content
068 * conforms to the server schema. It will obtain the schema by connecting to
069 * the server and retrieving the default schema (i.e., the schema which governs
070 * the root DSE). By default, a thorough set of validation will be performed,
071 * but it is possible to disable certain types of validation.
072 * <BR><BR>
073 * Some of the APIs demonstrated by this example include:
074 * <UL>
075 * <LI>Argument Parsing (from the {@code com.unboundid.util.args}
076 * package)</LI>
077 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
078 * package)</LI>
079 * <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
080 * <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
081 * package)</LI>
082 * </UL>
083 * <BR><BR>
084 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
085 * class (to obtain the information to use to connect to the server to read the
086 * schema), as well as the following additional arguments:
087 * <UL>
088 * <LI>"--schemaDirectory {path}" -- specifies the path to a directory
089 * containing files with schema definitions. If this argument is
090 * provided, then no attempt will be made to communicate with a directory
091 * server.</LI>
092 * <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
093 * file to be validated.</LI>
094 * <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
095 * compressed.</LI>
096 * <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
097 * to be written with information about all entries that failed
098 * validation.</LI>
099 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
100 * concurrent threads to use when processing the LDIF. If this is not
101 * provided, then a default of one thread will be used.</LI>
102 * <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
103 * process should ignore validation failures due to entries that contain
104 * object classes not defined in the server schema.</LI>
105 * <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
106 * should ignore validation failures due to entries that contain
107 * attributes not defined in the server schema.</LI>
108 * <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
109 * ignore validation failures due to entries with malformed DNs.</LI>
110 * <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
111 * should ignore validation failures due to entries that contain an RDN
112 * attribute value that is not present in the set of entry
113 * attributes.</LI>
114 * <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
115 * process should ignore validation failures due to entries that either do
116 * not have a structural object class or that have multiple structural
117 * object classes.</LI>
118 * <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
119 * process should ignore validation failures due to entries containing
120 * auxiliary classes that are not allowed by a DIT content rule, or
121 * abstract classes that are not subclassed by an auxiliary or structural
122 * class contained in the entry.</LI>
123 * <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
124 * should ignore validation failures due to entries including attributes
125 * that are not allowed or are explicitly prohibited by a DIT content
126 * rule.</LI>
127 * <LI>"--ignoreMissingAttributes" -- indicates that the validation process
128 * should ignore validation failures due to entries missing required
129 * attributes.</LI>
130 * <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
131 * process should ignore validation failures due to single-valued
132 * attributes containing multiple values.</LI>
133 * <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
134 * should ignore validation failures due to attribute values which violate
135 * the associated attribute syntax.</LI>
136 * <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
137 * process should ignore validation failures due to attribute values which
138 * violate the associated attribute syntax, but only for the specified
139 * attribute types.</LI>
140 * <LI>"--ignoreNameForms" -- indicates that the validation process should
141 * ignore validation failures due to name form violations (in which the
142 * entry's RDN does not comply with the associated name form).</LI>
143 * </UL>
144 */
145 @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
146 public final class ValidateLDIF
147 extends LDAPCommandLineTool
148 implements LDIFReaderEntryTranslator
149 {
150 /**
151 * The end-of-line character for this platform.
152 */
153 private static final String EOL = System.getProperty("line.separator", "\n");
154
155
156
157 // The arguments used by this program.
158 private BooleanArgument ignoreDuplicateValues;
159 private BooleanArgument ignoreUndefinedObjectClasses;
160 private BooleanArgument ignoreUndefinedAttributes;
161 private BooleanArgument ignoreMalformedDNs;
162 private BooleanArgument ignoreMissingRDNValues;
163 private BooleanArgument ignoreMissingSuperiorObjectClasses;
164 private BooleanArgument ignoreStructuralObjectClasses;
165 private BooleanArgument ignoreProhibitedObjectClasses;
166 private BooleanArgument ignoreProhibitedAttributes;
167 private BooleanArgument ignoreMissingAttributes;
168 private BooleanArgument ignoreSingleValuedAttributes;
169 private BooleanArgument ignoreAttributeSyntax;
170 private BooleanArgument ignoreNameForms;
171 private BooleanArgument isCompressed;
172 private FileArgument schemaDirectory;
173 private FileArgument ldifFile;
174 private FileArgument rejectFile;
175 private IntegerArgument numThreads;
176 private StringArgument ignoreSyntaxViolationsForAttribute;
177
178 // The counter used to keep track of the number of entries processed.
179 private final AtomicLong entriesProcessed = new AtomicLong(0L);
180
181 // The counter used to keep track of the number of entries that could not be
182 // parsed as valid entries.
183 private final AtomicLong malformedEntries = new AtomicLong(0L);
184
185 // The entry validator that will be used to validate the entries.
186 private EntryValidator entryValidator;
187
188 // The LDIF writer that will be used to write rejected entries.
189 private LDIFWriter rejectWriter;
190
191
192
193 /**
194 * Parse the provided command line arguments and make the appropriate set of
195 * changes.
196 *
197 * @param args The command line arguments provided to this program.
198 */
199 public static void main(final String[] args)
200 {
201 final ResultCode resultCode = main(args, System.out, System.err);
202 if (resultCode != ResultCode.SUCCESS)
203 {
204 System.exit(resultCode.intValue());
205 }
206 }
207
208
209
210 /**
211 * Parse the provided command line arguments and make the appropriate set of
212 * changes.
213 *
214 * @param args The command line arguments provided to this program.
215 * @param outStream The output stream to which standard out should be
216 * written. It may be {@code null} if output should be
217 * suppressed.
218 * @param errStream The output stream to which standard error should be
219 * written. It may be {@code null} if error messages
220 * should be suppressed.
221 *
222 * @return A result code indicating whether the processing was successful.
223 */
224 public static ResultCode main(final String[] args,
225 final OutputStream outStream,
226 final OutputStream errStream)
227 {
228 final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
229 return validateLDIF.runTool(args);
230 }
231
232
233
234 /**
235 * Creates a new instance of this tool.
236 *
237 * @param outStream The output stream to which standard out should be
238 * written. It may be {@code null} if output should be
239 * suppressed.
240 * @param errStream The output stream to which standard error should be
241 * written. It may be {@code null} if error messages
242 * should be suppressed.
243 */
244 public ValidateLDIF(final OutputStream outStream,
245 final OutputStream errStream)
246 {
247 super(outStream, errStream);
248 }
249
250
251
252 /**
253 * Retrieves the name for this tool.
254 *
255 * @return The name for this tool.
256 */
257 @Override()
258 public String getToolName()
259 {
260 return "validate-ldif";
261 }
262
263
264
265 /**
266 * Retrieves the description for this tool.
267 *
268 * @return The description for this tool.
269 */
270 @Override()
271 public String getToolDescription()
272 {
273 return "Validate the contents of an LDIF file " +
274 "against the server schema.";
275 }
276
277
278
279 /**
280 * Retrieves the version string for this tool.
281 *
282 * @return The version string for this tool.
283 */
284 @Override()
285 public String getToolVersion()
286 {
287 return Version.NUMERIC_VERSION_STRING;
288 }
289
290
291
292 /**
293 * Indicates whether this tool should provide support for an interactive mode,
294 * in which the tool offers a mode in which the arguments can be provided in
295 * a text-driven menu rather than requiring them to be given on the command
296 * line. If interactive mode is supported, it may be invoked using the
297 * "--interactive" argument. Alternately, if interactive mode is supported
298 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
299 * interactive mode may be invoked by simply launching the tool without any
300 * arguments.
301 *
302 * @return {@code true} if this tool supports interactive mode, or
303 * {@code false} if not.
304 */
305 @Override()
306 public boolean supportsInteractiveMode()
307 {
308 return true;
309 }
310
311
312
313 /**
314 * Indicates whether this tool defaults to launching in interactive mode if
315 * the tool is invoked without any command-line arguments. This will only be
316 * used if {@link #supportsInteractiveMode()} returns {@code true}.
317 *
318 * @return {@code true} if this tool defaults to using interactive mode if
319 * launched without any command-line arguments, or {@code false} if
320 * not.
321 */
322 @Override()
323 public boolean defaultsToInteractiveMode()
324 {
325 return true;
326 }
327
328
329
330 /**
331 * Indicates whether this tool should provide arguments for redirecting output
332 * to a file. If this method returns {@code true}, then the tool will offer
333 * an "--outputFile" argument that will specify the path to a file to which
334 * all standard output and standard error content will be written, and it will
335 * also offer a "--teeToStandardOut" argument that can only be used if the
336 * "--outputFile" argument is present and will cause all output to be written
337 * to both the specified output file and to standard output.
338 *
339 * @return {@code true} if this tool should provide arguments for redirecting
340 * output to a file, or {@code false} if not.
341 */
342 @Override()
343 protected boolean supportsOutputFile()
344 {
345 return true;
346 }
347
348
349
350 /**
351 * Indicates whether this tool should default to interactively prompting for
352 * the bind password if a password is required but no argument was provided
353 * to indicate how to get the password.
354 *
355 * @return {@code true} if this tool should default to interactively
356 * prompting for the bind password, or {@code false} if not.
357 */
358 @Override()
359 protected boolean defaultToPromptForBindPassword()
360 {
361 return true;
362 }
363
364
365
366 /**
367 * Indicates whether this tool supports the use of a properties file for
368 * specifying default values for arguments that aren't specified on the
369 * command line.
370 *
371 * @return {@code true} if this tool supports the use of a properties file
372 * for specifying default values for arguments that aren't specified
373 * on the command line, or {@code false} if not.
374 */
375 @Override()
376 public boolean supportsPropertiesFile()
377 {
378 return true;
379 }
380
381
382
383 /**
384 * Indicates whether the LDAP-specific arguments should include alternate
385 * versions of all long identifiers that consist of multiple words so that
386 * they are available in both camelCase and dash-separated versions.
387 *
388 * @return {@code true} if this tool should provide multiple versions of
389 * long identifiers for LDAP-specific arguments, or {@code false} if
390 * not.
391 */
392 @Override()
393 protected boolean includeAlternateLongIdentifiers()
394 {
395 return true;
396 }
397
398
399
400 /**
401 * Adds the arguments used by this program that aren't already provided by the
402 * generic {@code LDAPCommandLineTool} framework.
403 *
404 * @param parser The argument parser to which the arguments should be added.
405 *
406 * @throws ArgumentException If a problem occurs while adding the arguments.
407 */
408 @Override()
409 public void addNonLDAPArguments(final ArgumentParser parser)
410 throws ArgumentException
411 {
412 String description = "The path to the LDIF file to process.";
413 ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
414 true, true, true, false);
415 ldifFile.addLongIdentifier("ldif-file");
416 parser.addArgument(ldifFile);
417
418 description = "Indicates that the specified LDIF file is compressed " +
419 "using gzip compression.";
420 isCompressed = new BooleanArgument('c', "isCompressed", description);
421 isCompressed.addLongIdentifier("is-compressed");
422 parser.addArgument(isCompressed);
423
424 description = "The path to the file to which rejected entries should be " +
425 "written.";
426 rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
427 description, false, true, true, false);
428 rejectFile.addLongIdentifier("reject-file");
429 parser.addArgument(rejectFile);
430
431 description = "The path to a directory containing one or more LDIF files " +
432 "with the schema information to use. If this is provided, " +
433 "then no LDAP communication will be performed.";
434 schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
435 "{path}", description, true, true, false, true);
436 schemaDirectory.addLongIdentifier("schema-directory");
437 parser.addArgument(schemaDirectory);
438
439 description = "The number of threads to use when processing the LDIF file.";
440 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
441 description, 1, Integer.MAX_VALUE, 1);
442 numThreads.addLongIdentifier("num-threads");
443 parser.addArgument(numThreads);
444
445 description = "Ignore validation failures due to entries containing " +
446 "duplicate values for the same attribute.";
447 ignoreDuplicateValues =
448 new BooleanArgument(null, "ignoreDuplicateValues", description);
449 ignoreDuplicateValues.setArgumentGroupName(
450 "Validation Strictness Arguments");
451 ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values");
452 parser.addArgument(ignoreDuplicateValues);
453
454 description = "Ignore validation failures due to object classes not " +
455 "defined in the schema.";
456 ignoreUndefinedObjectClasses =
457 new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
458 ignoreUndefinedObjectClasses.setArgumentGroupName(
459 "Validation Strictness Arguments");
460 ignoreUndefinedObjectClasses.addLongIdentifier(
461 "ignore-undefined-object-classes");
462 parser.addArgument(ignoreUndefinedObjectClasses);
463
464 description = "Ignore validation failures due to attributes not defined " +
465 "in the schema.";
466 ignoreUndefinedAttributes =
467 new BooleanArgument(null, "ignoreUndefinedAttributes", description);
468 ignoreUndefinedAttributes.setArgumentGroupName(
469 "Validation Strictness Arguments");
470 ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes");
471 parser.addArgument(ignoreUndefinedAttributes);
472
473 description = "Ignore validation failures due to entries with malformed " +
474 "DNs.";
475 ignoreMalformedDNs =
476 new BooleanArgument(null, "ignoreMalformedDNs", description);
477 ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
478 ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns");
479 parser.addArgument(ignoreMalformedDNs);
480
481 description = "Ignore validation failures due to entries with RDN " +
482 "attribute values that are missing from the set of entry " +
483 "attributes.";
484 ignoreMissingRDNValues =
485 new BooleanArgument(null, "ignoreMissingRDNValues", description);
486 ignoreMissingRDNValues.setArgumentGroupName(
487 "Validation Strictness Arguments");
488 ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values");
489 parser.addArgument(ignoreMissingRDNValues);
490
491 description = "Ignore validation failures due to entries without exactly " +
492 "structural object class.";
493 ignoreStructuralObjectClasses =
494 new BooleanArgument(null, "ignoreStructuralObjectClasses",
495 description);
496 ignoreStructuralObjectClasses.setArgumentGroupName(
497 "Validation Strictness Arguments");
498 ignoreStructuralObjectClasses.addLongIdentifier(
499 "ignore-structural-object-classes");
500 parser.addArgument(ignoreStructuralObjectClasses);
501
502 description = "Ignore validation failures due to entries with object " +
503 "classes that are not allowed.";
504 ignoreProhibitedObjectClasses =
505 new BooleanArgument(null, "ignoreProhibitedObjectClasses",
506 description);
507 ignoreProhibitedObjectClasses.setArgumentGroupName(
508 "Validation Strictness Arguments");
509 ignoreProhibitedObjectClasses.addLongIdentifier(
510 "ignore-prohibited-object-classes");
511 parser.addArgument(ignoreProhibitedObjectClasses);
512
513 description = "Ignore validation failures due to entries that are " +
514 "one or more superior object classes.";
515 ignoreMissingSuperiorObjectClasses =
516 new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
517 description);
518 ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
519 "Validation Strictness Arguments");
520 ignoreMissingSuperiorObjectClasses.addLongIdentifier(
521 "ignore-missing-superior-object-classes");
522 parser.addArgument(ignoreMissingSuperiorObjectClasses);
523
524 description = "Ignore validation failures due to entries with attributes " +
525 "that are not allowed.";
526 ignoreProhibitedAttributes =
527 new BooleanArgument(null, "ignoreProhibitedAttributes", description);
528 ignoreProhibitedAttributes.setArgumentGroupName(
529 "Validation Strictness Arguments");
530 ignoreProhibitedAttributes.addLongIdentifier(
531 "ignore-prohibited-attributes");
532 parser.addArgument(ignoreProhibitedAttributes);
533
534 description = "Ignore validation failures due to entries missing " +
535 "required attributes.";
536 ignoreMissingAttributes =
537 new BooleanArgument(null, "ignoreMissingAttributes", description);
538 ignoreMissingAttributes.setArgumentGroupName(
539 "Validation Strictness Arguments");
540 ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes");
541 parser.addArgument(ignoreMissingAttributes);
542
543 description = "Ignore validation failures due to entries with multiple " +
544 "values for single-valued attributes.";
545 ignoreSingleValuedAttributes =
546 new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
547 ignoreSingleValuedAttributes.setArgumentGroupName(
548 "Validation Strictness Arguments");
549 ignoreSingleValuedAttributes.addLongIdentifier(
550 "ignore-single-valued-attributes");
551 parser.addArgument(ignoreSingleValuedAttributes);
552
553 description = "Ignore validation failures due to entries with attribute " +
554 "values that violate their associated syntax. If this is " +
555 "provided, then no attribute syntax violations will be " +
556 "flagged. If this is not provided, then all attribute " +
557 "syntax violations will be flagged except for violations " +
558 "in those attributes excluded by the " +
559 "--ignoreSyntaxViolationsForAttribute argument.";
560 ignoreAttributeSyntax =
561 new BooleanArgument(null, "ignoreAttributeSyntax", description);
562 ignoreAttributeSyntax.setArgumentGroupName(
563 "Validation Strictness Arguments");
564 ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax");
565 parser.addArgument(ignoreAttributeSyntax);
566
567 description = "The name or OID of an attribute for which to ignore " +
568 "validation failures due to violations of the associated " +
569 "attribute syntax. This argument can only be used if the " +
570 "--ignoreAttributeSyntax argument is not provided.";
571 ignoreSyntaxViolationsForAttribute = new StringArgument(null,
572 "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
573 ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
574 "Validation Strictness Arguments");
575 ignoreSyntaxViolationsForAttribute.addLongIdentifier(
576 "ignore-syntax-violations-for-attribute");
577 parser.addArgument(ignoreSyntaxViolationsForAttribute);
578
579 description = "Ignore validation failures due to entries with RDNs " +
580 "that violate the associated name form definition.";
581 ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
582 ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
583 ignoreNameForms.addLongIdentifier("ignore-name-forms");
584 parser.addArgument(ignoreNameForms);
585
586
587 // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
588 // cannot be used together.
589 parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
590 ignoreSyntaxViolationsForAttribute);
591 }
592
593
594
595 /**
596 * Performs the actual processing for this tool. In this case, it gets a
597 * connection to the directory server and uses it to retrieve the server
598 * schema. It then reads the LDIF file and validates each entry accordingly.
599 *
600 * @return The result code for the processing that was performed.
601 */
602 @Override()
603 public ResultCode doToolProcessing()
604 {
605 // Get the connection to the directory server and use it to read the schema.
606 final Schema schema;
607 if (schemaDirectory.isPresent())
608 {
609 final File schemaDir = schemaDirectory.getValue();
610
611 try
612 {
613 final TreeMap<String,File> fileMap = new TreeMap<String,File>();
614 for (final File f : schemaDir.listFiles())
615 {
616 final String name = f.getName();
617 if (f.isFile() && name.endsWith(".ldif"))
618 {
619 fileMap.put(name, f);
620 }
621 }
622
623 if (fileMap.isEmpty())
624 {
625 err("No LDIF files found in directory " +
626 schemaDir.getAbsolutePath());
627 return ResultCode.PARAM_ERROR;
628 }
629
630 final ArrayList<File> fileList = new ArrayList<File>(fileMap.values());
631 schema = Schema.getSchema(fileList);
632 }
633 catch (Exception e)
634 {
635 err("Unable to read schema from files in directory " +
636 schemaDir.getAbsolutePath() + ": " + getExceptionMessage(e));
637 return ResultCode.LOCAL_ERROR;
638 }
639 }
640 else
641 {
642 try
643 {
644 final LDAPConnection connection = getConnection();
645 schema = connection.getSchema();
646 connection.close();
647 }
648 catch (LDAPException le)
649 {
650 err("Unable to connect to the directory server and read the schema: ",
651 le.getMessage());
652 return le.getResultCode();
653 }
654 }
655
656
657 // Create the entry validator and initialize its configuration.
658 entryValidator = new EntryValidator(schema);
659 entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
660 entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
661 entryValidator.setCheckEntryMissingRDNValues(
662 !ignoreMissingRDNValues.isPresent());
663 entryValidator.setCheckMissingAttributes(
664 !ignoreMissingAttributes.isPresent());
665 entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
666 entryValidator.setCheckProhibitedAttributes(
667 !ignoreProhibitedAttributes.isPresent());
668 entryValidator.setCheckProhibitedObjectClasses(
669 !ignoreProhibitedObjectClasses.isPresent());
670 entryValidator.setCheckMissingSuperiorObjectClasses(
671 !ignoreMissingSuperiorObjectClasses.isPresent());
672 entryValidator.setCheckSingleValuedAttributes(
673 !ignoreSingleValuedAttributes.isPresent());
674 entryValidator.setCheckStructuralObjectClasses(
675 !ignoreStructuralObjectClasses.isPresent());
676 entryValidator.setCheckUndefinedAttributes(
677 !ignoreUndefinedAttributes.isPresent());
678 entryValidator.setCheckUndefinedObjectClasses(
679 !ignoreUndefinedObjectClasses.isPresent());
680
681 if (ignoreSyntaxViolationsForAttribute.isPresent())
682 {
683 entryValidator.setIgnoreSyntaxViolationAttributeTypes(
684 ignoreSyntaxViolationsForAttribute.getValues());
685 }
686
687
688 // Create an LDIF reader that can be used to read through the LDIF file.
689 final LDIFReader ldifReader;
690 rejectWriter = null;
691 try
692 {
693 InputStream inputStream = new FileInputStream(ldifFile.getValue());
694 if (isCompressed.isPresent())
695 {
696 inputStream = new GZIPInputStream(inputStream);
697 }
698 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
699 }
700 catch (Exception e)
701 {
702 err("Unable to open the LDIF reader: ", getExceptionMessage(e));
703 return ResultCode.LOCAL_ERROR;
704 }
705
706 ldifReader.setSchema(schema);
707 if (ignoreDuplicateValues.isPresent())
708 {
709 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
710 }
711 else
712 {
713 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
714 }
715
716 try
717 {
718 // Create an LDIF writer that can be used to write information about
719 // rejected entries.
720 try
721 {
722 if (rejectFile.isPresent())
723 {
724 rejectWriter = new LDIFWriter(rejectFile.getValue());
725 }
726 }
727 catch (Exception e)
728 {
729 err("Unable to create the reject writer: ", getExceptionMessage(e));
730 return ResultCode.LOCAL_ERROR;
731 }
732
733 ResultCode resultCode = ResultCode.SUCCESS;
734 while (true)
735 {
736 try
737 {
738 final Entry e = ldifReader.readEntry();
739 if (e == null)
740 {
741 // Because we're performing parallel processing and returning null
742 // from the translate method, LDIFReader.readEntry() should never
743 // return a non-null value. However, it can throw an LDIFException
744 // if it encounters an invalid entry, or an IOException if there's
745 // a problem reading from the file, so we should still iterate
746 // through all of the entries to catch and report on those problems.
747 break;
748 }
749 }
750 catch (LDIFException le)
751 {
752 malformedEntries.incrementAndGet();
753
754 if (resultCode == ResultCode.SUCCESS)
755 {
756 resultCode = ResultCode.DECODING_ERROR;
757 }
758
759 if (rejectWriter != null)
760 {
761 try
762 {
763 rejectWriter.writeComment(
764 "Unable to parse an entry read from LDIF:", false, false);
765 if (le.mayContinueReading())
766 {
767 rejectWriter.writeComment(getExceptionMessage(le), false, true);
768 }
769 else
770 {
771 rejectWriter.writeComment(getExceptionMessage(le), false,
772 false);
773 rejectWriter.writeComment("Unable to continue LDIF processing.",
774 false, true);
775 err("Aborting LDIF processing: ", getExceptionMessage(le));
776 return ResultCode.LOCAL_ERROR;
777 }
778 }
779 catch (IOException ioe)
780 {
781 err("Unable to write to the reject file:",
782 getExceptionMessage(ioe));
783 err("LDIF parse failure that triggered the rejection: ",
784 getExceptionMessage(le));
785 return ResultCode.LOCAL_ERROR;
786 }
787 }
788 }
789 catch (IOException ioe)
790 {
791
792 if (rejectWriter != null)
793 {
794 try
795 {
796 rejectWriter.writeComment("I/O error reading from LDIF:", false,
797 false);
798 rejectWriter.writeComment(getExceptionMessage(ioe), false,
799 true);
800 return ResultCode.LOCAL_ERROR;
801 }
802 catch (Exception ex)
803 {
804 err("I/O error reading from LDIF:", getExceptionMessage(ioe));
805 return ResultCode.LOCAL_ERROR;
806 }
807 }
808 }
809 }
810
811 if (malformedEntries.get() > 0)
812 {
813 out(malformedEntries.get() + " entries were malformed and could not " +
814 "be read from the LDIF file.");
815 }
816
817 if (entryValidator.getInvalidEntries() > 0)
818 {
819 if (resultCode == ResultCode.SUCCESS)
820 {
821 resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
822 }
823
824 for (final String s : entryValidator.getInvalidEntrySummary(true))
825 {
826 out(s);
827 }
828 }
829 else
830 {
831 if (malformedEntries.get() == 0)
832 {
833 out("No errors were encountered.");
834 }
835 }
836
837 return resultCode;
838 }
839 finally
840 {
841 try
842 {
843 ldifReader.close();
844 }
845 catch (Exception e) {}
846
847 try
848 {
849 if (rejectWriter != null)
850 {
851 rejectWriter.close();
852 }
853 }
854 catch (Exception e) {}
855 }
856 }
857
858
859
860 /**
861 * Examines the provided entry to determine whether it conforms to the
862 * server schema.
863 *
864 * @param entry The entry to be examined.
865 * @param firstLineNumber The line number of the LDIF source on which the
866 * provided entry begins.
867 *
868 * @return The updated entry. This method will always return {@code null}
869 * because all of the real processing needed for the entry is
870 * performed in this method and the entry isn't needed any more
871 * after this method is done.
872 */
873 public Entry translate(final Entry entry, final long firstLineNumber)
874 {
875 final ArrayList<String> invalidReasons = new ArrayList<String>(5);
876 if (! entryValidator.entryIsValid(entry, invalidReasons))
877 {
878 if (rejectWriter != null)
879 {
880 synchronized (this)
881 {
882 try
883 {
884 rejectWriter.writeEntry(entry, listToString(invalidReasons));
885 }
886 catch (IOException ioe) {}
887 }
888 }
889 }
890
891 final long numEntries = entriesProcessed.incrementAndGet();
892 if ((numEntries % 1000L) == 0L)
893 {
894 out("Processed ", numEntries, " entries.");
895 }
896
897 return null;
898 }
899
900
901
902 /**
903 * Converts the provided list of strings into a single string. It will
904 * contain line breaks after all but the last element.
905 *
906 * @param l The list of strings to convert to a single string.
907 *
908 * @return The string from the provided list, or {@code null} if the provided
909 * list is empty or {@code null}.
910 */
911 private static String listToString(final List<String> l)
912 {
913 if ((l == null) || (l.isEmpty()))
914 {
915 return null;
916 }
917
918 final StringBuilder buffer = new StringBuilder();
919 final Iterator<String> iterator = l.iterator();
920 while (iterator.hasNext())
921 {
922 buffer.append(iterator.next());
923 if (iterator.hasNext())
924 {
925 buffer.append(EOL);
926 }
927 }
928
929 return buffer.toString();
930 }
931
932
933
934 /**
935 * {@inheritDoc}
936 */
937 @Override()
938 public LinkedHashMap<String[],String> getExampleUsages()
939 {
940 final LinkedHashMap<String[],String> examples =
941 new LinkedHashMap<String[],String>(2);
942
943 String[] args =
944 {
945 "--hostname", "server.example.com",
946 "--port", "389",
947 "--ldifFile", "data.ldif",
948 "--rejectFile", "rejects.ldif",
949 "--numThreads", "4"
950 };
951 String description =
952 "Validate the contents of the 'data.ldif' file using the schema " +
953 "defined in the specified directory server using four concurrent " +
954 "threads. All types of validation will be performed, and " +
955 "information about any errors will be written to the 'rejects.ldif' " +
956 "file.";
957 examples.put(args, description);
958
959
960 args = new String[]
961 {
962 "--schemaDirectory", "/ds/config/schema",
963 "--ldifFile", "data.ldif",
964 "--rejectFile", "rejects.ldif",
965 "--ignoreStructuralObjectClasses",
966 "--ignoreAttributeSyntax"
967 };
968 description =
969 "Validate the contents of the 'data.ldif' file using the schema " +
970 "defined in LDIF files contained in the /ds/config/schema directory " +
971 "using a single thread. Any errors resulting from entries that do " +
972 "not have exactly one structural object class or from values which " +
973 "violate the syntax for their associated attribute types will be " +
974 "ignored. Information about any other failures will be written to " +
975 "the 'rejects.ldif' file.";
976 examples.put(args, description);
977
978 return examples;
979 }
980
981
982
983 /**
984 * @return EntryValidator
985 *
986 * Returns the EntryValidator
987 */
988 public EntryValidator getEntryValidator()
989 {
990 return entryValidator;
991 }
992 }