001 /*
002 * Copyright 2013-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005 /*
006 * Copyright (C) 2013-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021 package com.unboundid.ldap.sdk.examples;
022
023
024
025 import java.io.OutputStream;
026 import java.util.Collections;
027 import java.util.LinkedHashMap;
028 import java.util.List;
029 import java.util.Map;
030 import java.util.TreeMap;
031 import java.util.concurrent.atomic.AtomicLong;
032
033 import com.unboundid.asn1.ASN1OctetString;
034 import com.unboundid.ldap.sdk.Attribute;
035 import com.unboundid.ldap.sdk.DN;
036 import com.unboundid.ldap.sdk.Filter;
037 import com.unboundid.ldap.sdk.LDAPConnectionOptions;
038 import com.unboundid.ldap.sdk.LDAPConnectionPool;
039 import com.unboundid.ldap.sdk.LDAPException;
040 import com.unboundid.ldap.sdk.LDAPSearchException;
041 import com.unboundid.ldap.sdk.ResultCode;
042 import com.unboundid.ldap.sdk.SearchRequest;
043 import com.unboundid.ldap.sdk.SearchResult;
044 import com.unboundid.ldap.sdk.SearchResultEntry;
045 import com.unboundid.ldap.sdk.SearchResultReference;
046 import com.unboundid.ldap.sdk.SearchResultListener;
047 import com.unboundid.ldap.sdk.SearchScope;
048 import com.unboundid.ldap.sdk.Version;
049 import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
050 import com.unboundid.util.Debug;
051 import com.unboundid.util.LDAPCommandLineTool;
052 import com.unboundid.util.StaticUtils;
053 import com.unboundid.util.ThreadSafety;
054 import com.unboundid.util.ThreadSafetyLevel;
055 import com.unboundid.util.args.ArgumentException;
056 import com.unboundid.util.args.ArgumentParser;
057 import com.unboundid.util.args.DNArgument;
058 import com.unboundid.util.args.IntegerArgument;
059 import com.unboundid.util.args.StringArgument;
060
061
062
063 /**
064 * This class provides a tool that may be used to identify references to entries
065 * that do not exist. This tool can be useful for verifying existing data in
066 * directory servers that provide support for referential integrity.
067 * <BR><BR>
068 * All of the necessary information is provided using command line arguments.
069 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
070 * class, as well as the following additional arguments:
071 * <UL>
072 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
073 * for the searches. At least one base DN must be provided.</LI>
074 * <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
075 * that is expected to contain references to other entries. This
076 * attribute should be indexed for equality searches, and its values
077 * should be DNs. At least one attribute must be provided.</LI>
078 * <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
079 * to find entries with references to other entries should use the simple
080 * paged results control to iterate across entries in fixed-size pages
081 * rather than trying to use a single search to identify all entries that
082 * reference other entries.</LI>
083 * </UL>
084 */
085 @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
086 public final class IdentifyReferencesToMissingEntries
087 extends LDAPCommandLineTool
088 implements SearchResultListener
089 {
090 /**
091 * The serial version UID for this serializable class.
092 */
093 private static final long serialVersionUID = 1981894839719501258L;
094
095
096
097 // The number of entries examined so far.
098 private final AtomicLong entriesExamined;
099
100 // The argument used to specify the base DNs to use for searches.
101 private DNArgument baseDNArgument;
102
103 // The argument used to specify the search page size.
104 private IntegerArgument pageSizeArgument;
105
106 // The connection to use for retrieving referenced entries.
107 private LDAPConnectionPool getReferencedEntriesPool;
108
109 // A map with counts of missing references by attribute type.
110 private final Map<String,AtomicLong> missingReferenceCounts;
111
112 // The names of the attributes for which to find missing references.
113 private String[] attributes;
114
115 // The argument used to specify the attributes for which to find missing
116 // references.
117 private StringArgument attributeArgument;
118
119
120
121 /**
122 * Parse the provided command line arguments and perform the appropriate
123 * processing.
124 *
125 * @param args The command line arguments provided to this program.
126 */
127 public static void main(final String... args)
128 {
129 final ResultCode resultCode = main(args, System.out, System.err);
130 if (resultCode != ResultCode.SUCCESS)
131 {
132 System.exit(resultCode.intValue());
133 }
134 }
135
136
137
138 /**
139 * Parse the provided command line arguments and perform the appropriate
140 * processing.
141 *
142 * @param args The command line arguments provided to this program.
143 * @param outStream The output stream to which standard out should be
144 * written. It may be {@code null} if output should be
145 * suppressed.
146 * @param errStream The output stream to which standard error should be
147 * written. It may be {@code null} if error messages
148 * should be suppressed.
149 *
150 * @return A result code indicating whether the processing was successful.
151 */
152 public static ResultCode main(final String[] args,
153 final OutputStream outStream,
154 final OutputStream errStream)
155 {
156 final IdentifyReferencesToMissingEntries tool =
157 new IdentifyReferencesToMissingEntries(outStream, errStream);
158 return tool.runTool(args);
159 }
160
161
162
163 /**
164 * Creates a new instance of this tool.
165 *
166 * @param outStream The output stream to which standard out should be
167 * written. It may be {@code null} if output should be
168 * suppressed.
169 * @param errStream The output stream to which standard error should be
170 * written. It may be {@code null} if error messages
171 * should be suppressed.
172 */
173 public IdentifyReferencesToMissingEntries(final OutputStream outStream,
174 final OutputStream errStream)
175 {
176 super(outStream, errStream);
177
178 baseDNArgument = null;
179 pageSizeArgument = null;
180 attributeArgument = null;
181 getReferencedEntriesPool = null;
182
183 entriesExamined = new AtomicLong(0L);
184 missingReferenceCounts = new TreeMap<String, AtomicLong>();
185 }
186
187
188
189 /**
190 * Retrieves the name of this tool. It should be the name of the command used
191 * to invoke this tool.
192 *
193 * @return The name for this tool.
194 */
195 @Override()
196 public String getToolName()
197 {
198 return "identify-references-to-missing-entries";
199 }
200
201
202
203 /**
204 * Retrieves a human-readable description for this tool.
205 *
206 * @return A human-readable description for this tool.
207 */
208 @Override()
209 public String getToolDescription()
210 {
211 return "This tool may be used to identify entries containing one or more " +
212 "attributes which reference entries that do not exist. This may " +
213 "require the ability to perform unindexed searches and/or the " +
214 "ability to use the simple paged results control.";
215 }
216
217
218
219 /**
220 * Retrieves a version string for this tool, if available.
221 *
222 * @return A version string for this tool, or {@code null} if none is
223 * available.
224 */
225 @Override()
226 public String getToolVersion()
227 {
228 return Version.NUMERIC_VERSION_STRING;
229 }
230
231
232
233 /**
234 * Indicates whether this tool should provide support for an interactive mode,
235 * in which the tool offers a mode in which the arguments can be provided in
236 * a text-driven menu rather than requiring them to be given on the command
237 * line. If interactive mode is supported, it may be invoked using the
238 * "--interactive" argument. Alternately, if interactive mode is supported
239 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
240 * interactive mode may be invoked by simply launching the tool without any
241 * arguments.
242 *
243 * @return {@code true} if this tool supports interactive mode, or
244 * {@code false} if not.
245 */
246 @Override()
247 public boolean supportsInteractiveMode()
248 {
249 return true;
250 }
251
252
253
254 /**
255 * Indicates whether this tool defaults to launching in interactive mode if
256 * the tool is invoked without any command-line arguments. This will only be
257 * used if {@link #supportsInteractiveMode()} returns {@code true}.
258 *
259 * @return {@code true} if this tool defaults to using interactive mode if
260 * launched without any command-line arguments, or {@code false} if
261 * not.
262 */
263 @Override()
264 public boolean defaultsToInteractiveMode()
265 {
266 return true;
267 }
268
269
270
271 /**
272 * Indicates whether this tool should provide arguments for redirecting output
273 * to a file. If this method returns {@code true}, then the tool will offer
274 * an "--outputFile" argument that will specify the path to a file to which
275 * all standard output and standard error content will be written, and it will
276 * also offer a "--teeToStandardOut" argument that can only be used if the
277 * "--outputFile" argument is present and will cause all output to be written
278 * to both the specified output file and to standard output.
279 *
280 * @return {@code true} if this tool should provide arguments for redirecting
281 * output to a file, or {@code false} if not.
282 */
283 @Override()
284 protected boolean supportsOutputFile()
285 {
286 return true;
287 }
288
289
290
291 /**
292 * Indicates whether this tool should default to interactively prompting for
293 * the bind password if a password is required but no argument was provided
294 * to indicate how to get the password.
295 *
296 * @return {@code true} if this tool should default to interactively
297 * prompting for the bind password, or {@code false} if not.
298 */
299 @Override()
300 protected boolean defaultToPromptForBindPassword()
301 {
302 return true;
303 }
304
305
306
307 /**
308 * Indicates whether this tool supports the use of a properties file for
309 * specifying default values for arguments that aren't specified on the
310 * command line.
311 *
312 * @return {@code true} if this tool supports the use of a properties file
313 * for specifying default values for arguments that aren't specified
314 * on the command line, or {@code false} if not.
315 */
316 @Override()
317 public boolean supportsPropertiesFile()
318 {
319 return true;
320 }
321
322
323
324 /**
325 * Indicates whether the LDAP-specific arguments should include alternate
326 * versions of all long identifiers that consist of multiple words so that
327 * they are available in both camelCase and dash-separated versions.
328 *
329 * @return {@code true} if this tool should provide multiple versions of
330 * long identifiers for LDAP-specific arguments, or {@code false} if
331 * not.
332 */
333 @Override()
334 protected boolean includeAlternateLongIdentifiers()
335 {
336 return true;
337 }
338
339
340
341 /**
342 * Adds the arguments needed by this command-line tool to the provided
343 * argument parser which are not related to connecting or authenticating to
344 * the directory server.
345 *
346 * @param parser The argument parser to which the arguments should be added.
347 *
348 * @throws ArgumentException If a problem occurs while adding the arguments.
349 */
350 @Override()
351 public void addNonLDAPArguments(final ArgumentParser parser)
352 throws ArgumentException
353 {
354 String description = "The search base DN(s) to use to find entries with " +
355 "references to other entries. At least one base DN must be " +
356 "specified.";
357 baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
358 description);
359 baseDNArgument.addLongIdentifier("base-dn");
360 parser.addArgument(baseDNArgument);
361
362 description = "The attribute(s) for which to find missing references. " +
363 "At least one attribute must be specified, and each attribute " +
364 "must be indexed for equality searches and have values which are DNs.";
365 attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
366 description);
367 parser.addArgument(attributeArgument);
368
369 description = "The maximum number of entries to retrieve at a time when " +
370 "attempting to find entries with references to other entries. This " +
371 "requires that the authenticated user have permission to use the " +
372 "simple paged results control, but it can avoid problems with the " +
373 "server sending entries too quickly for the client to handle. By " +
374 "default, the simple paged results control will not be used.";
375 pageSizeArgument =
376 new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
377 description, 1, Integer.MAX_VALUE);
378 pageSizeArgument.addLongIdentifier("simple-page-size");
379 parser.addArgument(pageSizeArgument);
380 }
381
382
383
384 /**
385 * Retrieves the connection options that should be used for connections that
386 * are created with this command line tool. Subclasses may override this
387 * method to use a custom set of connection options.
388 *
389 * @return The connection options that should be used for connections that
390 * are created with this command line tool.
391 */
392 @Override()
393 public LDAPConnectionOptions getConnectionOptions()
394 {
395 final LDAPConnectionOptions options = new LDAPConnectionOptions();
396
397 options.setUseSynchronousMode(true);
398 options.setResponseTimeoutMillis(0L);
399
400 return options;
401 }
402
403
404
405 /**
406 * Performs the core set of processing for this tool.
407 *
408 * @return A result code that indicates whether the processing completed
409 * successfully.
410 */
411 @Override()
412 public ResultCode doToolProcessing()
413 {
414 // Establish a connection to the target directory server to use for
415 // finding references to entries.
416 final LDAPConnectionPool findReferencesPool;
417 try
418 {
419 findReferencesPool = getConnectionPool(1, 1);
420 findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true);
421 }
422 catch (final LDAPException le)
423 {
424 Debug.debugException(le);
425 err("Unable to establish a connection to the directory server: ",
426 StaticUtils.getExceptionMessage(le));
427 return le.getResultCode();
428 }
429
430 try
431 {
432 // Establish a second connection to use for retrieving referenced entries.
433 try
434 {
435 getReferencedEntriesPool = getConnectionPool(1,1);
436 getReferencedEntriesPool.
437 setRetryFailedOperationsDueToInvalidConnections(true);
438 }
439 catch (final LDAPException le)
440 {
441 Debug.debugException(le);
442 err("Unable to establish a connection to the directory server: ",
443 StaticUtils.getExceptionMessage(le));
444 return le.getResultCode();
445 }
446
447
448 // Get the set of attributes for which to find missing references.
449 final List<String> attrList = attributeArgument.getValues();
450 attributes = new String[attrList.size()];
451 attrList.toArray(attributes);
452
453
454 // Construct a search filter that will be used to find all entries with
455 // references to other entries.
456 final Filter filter;
457 if (attributes.length == 1)
458 {
459 filter = Filter.createPresenceFilter(attributes[0]);
460 missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
461 }
462 else
463 {
464 final Filter[] orComps = new Filter[attributes.length];
465 for (int i=0; i < attributes.length; i++)
466 {
467 orComps[i] = Filter.createPresenceFilter(attributes[i]);
468 missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
469 }
470 filter = Filter.createORFilter(orComps);
471 }
472
473
474 // Iterate across all of the search base DNs and perform searches to find
475 // missing references.
476 for (final DN baseDN : baseDNArgument.getValues())
477 {
478 ASN1OctetString cookie = null;
479 do
480 {
481 final SearchRequest searchRequest = new SearchRequest(this,
482 baseDN.toString(), SearchScope.SUB, filter, attributes);
483 if (pageSizeArgument.isPresent())
484 {
485 searchRequest.addControl(new SimplePagedResultsControl(
486 pageSizeArgument.getValue(), cookie, false));
487 }
488
489 SearchResult searchResult;
490 try
491 {
492 searchResult = findReferencesPool.search(searchRequest);
493 }
494 catch (final LDAPSearchException lse)
495 {
496 Debug.debugException(lse);
497 try
498 {
499 searchResult = findReferencesPool.search(searchRequest);
500 }
501 catch (final LDAPSearchException lse2)
502 {
503 Debug.debugException(lse2);
504 searchResult = lse2.getSearchResult();
505 }
506 }
507
508 if (searchResult.getResultCode() != ResultCode.SUCCESS)
509 {
510 err("An error occurred while attempting to search for missing " +
511 "references to entries below " + baseDN + ": " +
512 searchResult.getDiagnosticMessage());
513 return searchResult.getResultCode();
514 }
515
516 final SimplePagedResultsControl pagedResultsResponse;
517 try
518 {
519 pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
520 }
521 catch (final LDAPException le)
522 {
523 Debug.debugException(le);
524 err("An error occurred while attempting to decode a simple " +
525 "paged results response control in the response to a " +
526 "search for entries below " + baseDN + ": " +
527 StaticUtils.getExceptionMessage(le));
528 return le.getResultCode();
529 }
530
531 if (pagedResultsResponse != null)
532 {
533 if (pagedResultsResponse.moreResultsToReturn())
534 {
535 cookie = pagedResultsResponse.getCookie();
536 }
537 else
538 {
539 cookie = null;
540 }
541 }
542 }
543 while (cookie != null);
544 }
545
546
547 // See if there were any missing references found.
548 boolean missingReferenceFound = false;
549 for (final Map.Entry<String,AtomicLong> e :
550 missingReferenceCounts.entrySet())
551 {
552 final long numMissing = e.getValue().get();
553 if (numMissing > 0L)
554 {
555 if (! missingReferenceFound)
556 {
557 err();
558 missingReferenceFound = true;
559 }
560
561 err("Found " + numMissing + ' ' + e.getKey() +
562 " references to entries that do not exist.");
563 }
564 }
565
566 if (missingReferenceFound)
567 {
568 return ResultCode.CONSTRAINT_VIOLATION;
569 }
570 else
571 {
572 out("No references were found to entries that do not exist.");
573 return ResultCode.SUCCESS;
574 }
575 }
576 finally
577 {
578 findReferencesPool.close();
579
580 if (getReferencedEntriesPool != null)
581 {
582 getReferencedEntriesPool.close();
583 }
584 }
585 }
586
587
588
589 /**
590 * Retrieves a map that correlates the number of missing references found by
591 * attribute type.
592 *
593 * @return A map that correlates the number of missing references found by
594 * attribute type.
595 */
596 public Map<String,AtomicLong> getMissingReferenceCounts()
597 {
598 return Collections.unmodifiableMap(missingReferenceCounts);
599 }
600
601
602
603 /**
604 * Retrieves a set of information that may be used to generate example usage
605 * information. Each element in the returned map should consist of a map
606 * between an example set of arguments and a string that describes the
607 * behavior of the tool when invoked with that set of arguments.
608 *
609 * @return A set of information that may be used to generate example usage
610 * information. It may be {@code null} or empty if no example usage
611 * information is available.
612 */
613 @Override()
614 public LinkedHashMap<String[],String> getExampleUsages()
615 {
616 final LinkedHashMap<String[],String> exampleMap =
617 new LinkedHashMap<String[],String>(1);
618
619 final String[] args =
620 {
621 "--hostname", "server.example.com",
622 "--port", "389",
623 "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
624 "--bindPassword", "password",
625 "--baseDN", "dc=example,dc=com",
626 "--attribute", "member",
627 "--attribute", "uniqueMember",
628 "--simplePageSize", "100"
629 };
630 exampleMap.put(args,
631 "Identify all entries below dc=example,dc=com in which either the " +
632 "member or uniqueMember attribute references an entry that " +
633 "does not exist.");
634
635 return exampleMap;
636 }
637
638
639
640 /**
641 * Indicates that the provided search result entry has been returned by the
642 * server and may be processed by this search result listener.
643 *
644 * @param searchEntry The search result entry that has been returned by the
645 * server.
646 */
647 public void searchEntryReturned(final SearchResultEntry searchEntry)
648 {
649 try
650 {
651 // Find attributes which references to entries that do not exist.
652 for (final String attr : attributes)
653 {
654 final List<Attribute> attrList =
655 searchEntry.getAttributesWithOptions(attr, null);
656 for (final Attribute a : attrList)
657 {
658 for (final String value : a.getValues())
659 {
660 try
661 {
662 final SearchResultEntry e =
663 getReferencedEntriesPool.getEntry(value, "1.1");
664 if (e == null)
665 {
666 err("Entry '", searchEntry.getDN(), "' includes attribute ",
667 a.getName(), " that references entry '", value,
668 "' which does not exist.");
669 missingReferenceCounts.get(attr).incrementAndGet();
670 }
671 }
672 catch (final LDAPException le)
673 {
674 Debug.debugException(le);
675 err("An error occurred while attempting to determine whether " +
676 "entry '" + value + "' referenced in attribute " +
677 a.getName() + " of entry '" + searchEntry.getDN() +
678 "' exists: " + StaticUtils.getExceptionMessage(le));
679 missingReferenceCounts.get(attr).incrementAndGet();
680 }
681 }
682 }
683 }
684 }
685 finally
686 {
687 final long count = entriesExamined.incrementAndGet();
688 if ((count % 1000L) == 0L)
689 {
690 out(count, " entries examined");
691 }
692 }
693 }
694
695
696
697 /**
698 * Indicates that the provided search result reference has been returned by
699 * the server and may be processed by this search result listener.
700 *
701 * @param searchReference The search result reference that has been returned
702 * by the server.
703 */
704 public void searchReferenceReturned(
705 final SearchResultReference searchReference)
706 {
707 // No implementation is required. This tool will not follow referrals.
708 }
709 }