001 /*
002 * Copyright 2010-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005 /*
006 * Copyright (C) 2010-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021 package com.unboundid.ldap.sdk.examples;
022
023
024
025 import java.io.IOException;
026 import java.io.OutputStream;
027 import java.io.Serializable;
028 import java.text.ParseException;
029 import java.util.LinkedHashMap;
030 import java.util.LinkedHashSet;
031 import java.util.List;
032 import java.util.Random;
033 import java.util.concurrent.CyclicBarrier;
034 import java.util.concurrent.atomic.AtomicBoolean;
035 import java.util.concurrent.atomic.AtomicLong;
036
037 import com.unboundid.ldap.sdk.LDAPConnection;
038 import com.unboundid.ldap.sdk.LDAPConnectionOptions;
039 import com.unboundid.ldap.sdk.LDAPException;
040 import com.unboundid.ldap.sdk.ResultCode;
041 import com.unboundid.ldap.sdk.SearchScope;
042 import com.unboundid.ldap.sdk.Version;
043 import com.unboundid.util.ColumnFormatter;
044 import com.unboundid.util.FixedRateBarrier;
045 import com.unboundid.util.FormattableColumn;
046 import com.unboundid.util.HorizontalAlignment;
047 import com.unboundid.util.LDAPCommandLineTool;
048 import com.unboundid.util.ObjectPair;
049 import com.unboundid.util.OutputFormat;
050 import com.unboundid.util.RateAdjustor;
051 import com.unboundid.util.ResultCodeCounter;
052 import com.unboundid.util.ThreadSafety;
053 import com.unboundid.util.ThreadSafetyLevel;
054 import com.unboundid.util.ValuePattern;
055 import com.unboundid.util.WakeableSleeper;
056 import com.unboundid.util.args.ArgumentException;
057 import com.unboundid.util.args.ArgumentParser;
058 import com.unboundid.util.args.BooleanArgument;
059 import com.unboundid.util.args.FileArgument;
060 import com.unboundid.util.args.IntegerArgument;
061 import com.unboundid.util.args.ScopeArgument;
062 import com.unboundid.util.args.StringArgument;
063
064 import static com.unboundid.util.Debug.*;
065 import static com.unboundid.util.StaticUtils.*;
066
067
068
069 /**
070 * This class provides a tool that can be used to search an LDAP directory
071 * server repeatedly using multiple threads, and then modify each entry
072 * returned by that server. It can help provide an estimate of the combined
073 * search and modify performance that a directory server is able to achieve.
074 * Either or both of the base DN and the search filter may be a value pattern as
075 * described in the {@link ValuePattern} class. This makes it possible to
076 * search over a range of entries rather than repeatedly performing searches
077 * with the same base DN and filter.
078 * <BR><BR>
079 * Some of the APIs demonstrated by this example include:
080 * <UL>
081 * <LI>Argument Parsing (from the {@code com.unboundid.util.args}
082 * package)</LI>
083 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
084 * package)</LI>
085 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
086 * package)</LI>
087 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
088 * </UL>
089 * <BR><BR>
090 * All of the necessary information is provided using command line arguments.
091 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
092 * class, as well as the following additional arguments:
093 * <UL>
094 * <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
095 * for the searches. This must be provided. It may be a simple DN, or it
096 * may be a value pattern to express a range of base DNs.</LI>
097 * <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
098 * search. The scope value should be one of "base", "one", "sub", or
099 * "subord". If this isn't specified, then a scope of "sub" will be
100 * used.</LI>
101 * <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
102 * the searches. This must be provided. It may be a simple filter, or it
103 * may be a value pattern to express a range of filters.</LI>
104 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
105 * attribute that should be included in entries returned from the server.
106 * If this is not provided, then all user attributes will be requested.
107 * This may include special tokens that the server may interpret, like
108 * "1.1" to indicate that no attributes should be returned, "*", for all
109 * user attributes, or "+" for all operational attributes. Multiple
110 * attributes may be requested with multiple instances of this
111 * argument.</LI>
112 * <LI>"-m {name}" or "--modifyAttribute {name}" -- specifies the name of the
113 * attribute to modify. Multiple attributes may be modified by providing
114 * multiple instances of this argument. At least one attribute must be
115 * provided.</LI>
116 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
117 * use for the values of the target attributes to modify. If this is not
118 * provided, then a default length of 10 bytes will be used.</LI>
119 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
120 * characters that will be used to generate the values to use for the
121 * target attributes to modify. It should only include ASCII characters.
122 * Values will be generated from randomly-selected characters from this
123 * set. If this is not provided, then a default set of lowercase
124 * alphabetic characters will be used.</LI>
125 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
126 * concurrent threads to use when performing the searches. If this is not
127 * provided, then a default of one thread will be used.</LI>
128 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
129 * time in seconds between lines out output. If this is not provided,
130 * then a default interval duration of five seconds will be used.</LI>
131 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
132 * intervals for which to run. If this is not provided, then it will
133 * run forever.</LI>
134 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
135 * iterations that should be performed on a connection before that
136 * connection is closed and replaced with a newly-established (and
137 * authenticated, if appropriate) connection.</LI>
138 * <LI>"-r {ops-per-second}" or "--ratePerSecond {ops-per-second}" --
139 * specifies the target number of operations to perform per second. Each
140 * search and modify operation will be counted separately for this
141 * purpose, so if a value of 1 is specified and a search returns two
142 * entries, then a total of three seconds will be required (one for the
143 * search and one for the modify for each entry). It is still necessary
144 * to specify a sufficient number of threads for achieving this rate. If
145 * this option is not provided, then the tool will run at the maximum rate
146 * for the specified number of threads.</LI>
147 * <LI>"--variableRateData {path}" -- specifies the path to a file containing
148 * information needed to allow the tool to vary the target rate over time.
149 * If this option is not provided, then the tool will either use a fixed
150 * target rate as specified by the "--ratePerSecond" argument, or it will
151 * run at the maximum rate.</LI>
152 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
153 * which sample data will be written illustrating and describing the
154 * format of the file expected to be used in conjunction with the
155 * "--variableRateData" argument.</LI>
156 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
157 * complete before beginning overall statistics collection.</LI>
158 * <LI>"--timestampFormat {format}" -- specifies the format to use for
159 * timestamps included before each output line. The format may be one of
160 * "none" (for no timestamps), "with-date" (to include both the date and
161 * the time), or "without-date" (to include only time time).</LI>
162 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
163 * authorization v2 control to request that the operations be processed
164 * using an alternate authorization identity. In this case, the bind DN
165 * should be that of a user that has permission to use this control. The
166 * authorization identity may be a value pattern.</LI>
167 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the
168 * result codes for failed operations should not be displayed.</LI>
169 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
170 * display-friendly format.</LI>
171 * </UL>
172 */
173 @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
174 public final class SearchAndModRate
175 extends LDAPCommandLineTool
176 implements Serializable
177 {
178 /**
179 * The serial version UID for this serializable class.
180 */
181 private static final long serialVersionUID = 3242469381380526294L;
182
183
184
185 // Indicates whether a request has been made to stop running.
186 private final AtomicBoolean stopRequested;
187
188 // The argument used to indicate whether to generate output in CSV format.
189 private BooleanArgument csvFormat;
190
191 // The argument used to indicate whether to suppress information about error
192 // result codes.
193 private BooleanArgument suppressErrors;
194
195 // The argument used to specify the collection interval.
196 private IntegerArgument collectionInterval;
197
198 // The argument used to specify the number of search and modify iterations on
199 // a connection before it is closed and re-established.
200 private IntegerArgument iterationsBeforeReconnect;
201
202 // The argument used to specify the number of intervals.
203 private IntegerArgument numIntervals;
204
205 // The argument used to specify the number of threads.
206 private IntegerArgument numThreads;
207
208 // The argument used to specify the seed to use for the random number
209 // generator.
210 private IntegerArgument randomSeed;
211
212 // The target rate of operations per second.
213 private IntegerArgument ratePerSecond;
214
215 // The argument used to specify a variable rate file.
216 private FileArgument sampleRateFile;
217
218 // The argument used to specify a variable rate file.
219 private FileArgument variableRateData;
220
221 // The argument used to specify the length of the values to generate.
222 private IntegerArgument valueLength;
223
224 // The number of warm-up intervals to perform.
225 private IntegerArgument warmUpIntervals;
226
227 // The argument used to specify the scope for the searches.
228 private ScopeArgument scopeArg;
229
230 // The argument used to specify the base DNs for the searches.
231 private StringArgument baseDN;
232
233 // The argument used to specify the set of characters to use when generating
234 // values.
235 private StringArgument characterSet;
236
237 // The argument used to specify the filters for the searches.
238 private StringArgument filter;
239
240 // The argument used to specify the attributes to modify.
241 private StringArgument modifyAttributes;
242
243 // The argument used to specify the proxied authorization identity.
244 private StringArgument proxyAs;
245
246 // The argument used to specify the attributes to return.
247 private StringArgument returnAttributes;
248
249 // The argument used to specify the timestamp format.
250 private StringArgument timestampFormat;
251
252 // The thread currently being used to run the searchrate tool.
253 private volatile Thread runningThread;
254
255 // A wakeable sleeper that will be used to sleep between reporting intervals.
256 private final WakeableSleeper sleeper;
257
258
259
260 /**
261 * Parse the provided command line arguments and make the appropriate set of
262 * changes.
263 *
264 * @param args The command line arguments provided to this program.
265 */
266 public static void main(final String[] args)
267 {
268 final ResultCode resultCode = main(args, System.out, System.err);
269 if (resultCode != ResultCode.SUCCESS)
270 {
271 System.exit(resultCode.intValue());
272 }
273 }
274
275
276
277 /**
278 * Parse the provided command line arguments and make the appropriate set of
279 * changes.
280 *
281 * @param args The command line arguments provided to this program.
282 * @param outStream The output stream to which standard out should be
283 * written. It may be {@code null} if output should be
284 * suppressed.
285 * @param errStream The output stream to which standard error should be
286 * written. It may be {@code null} if error messages
287 * should be suppressed.
288 *
289 * @return A result code indicating whether the processing was successful.
290 */
291 public static ResultCode main(final String[] args,
292 final OutputStream outStream,
293 final OutputStream errStream)
294 {
295 final SearchAndModRate searchAndModRate =
296 new SearchAndModRate(outStream, errStream);
297 return searchAndModRate.runTool(args);
298 }
299
300
301
302 /**
303 * Creates a new instance of this tool.
304 *
305 * @param outStream The output stream to which standard out should be
306 * written. It may be {@code null} if output should be
307 * suppressed.
308 * @param errStream The output stream to which standard error should be
309 * written. It may be {@code null} if error messages
310 * should be suppressed.
311 */
312 public SearchAndModRate(final OutputStream outStream,
313 final OutputStream errStream)
314 {
315 super(outStream, errStream);
316
317 stopRequested = new AtomicBoolean(false);
318 sleeper = new WakeableSleeper();
319 }
320
321
322
323 /**
324 * Retrieves the name for this tool.
325 *
326 * @return The name for this tool.
327 */
328 @Override()
329 public String getToolName()
330 {
331 return "search-and-mod-rate";
332 }
333
334
335
336 /**
337 * Retrieves the description for this tool.
338 *
339 * @return The description for this tool.
340 */
341 @Override()
342 public String getToolDescription()
343 {
344 return "Perform repeated searches against an " +
345 "LDAP directory server and modify each entry returned.";
346 }
347
348
349
350 /**
351 * Retrieves the version string for this tool.
352 *
353 * @return The version string for this tool.
354 */
355 @Override()
356 public String getToolVersion()
357 {
358 return Version.NUMERIC_VERSION_STRING;
359 }
360
361
362
363 /**
364 * Indicates whether this tool should provide support for an interactive mode,
365 * in which the tool offers a mode in which the arguments can be provided in
366 * a text-driven menu rather than requiring them to be given on the command
367 * line. If interactive mode is supported, it may be invoked using the
368 * "--interactive" argument. Alternately, if interactive mode is supported
369 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
370 * interactive mode may be invoked by simply launching the tool without any
371 * arguments.
372 *
373 * @return {@code true} if this tool supports interactive mode, or
374 * {@code false} if not.
375 */
376 @Override()
377 public boolean supportsInteractiveMode()
378 {
379 return true;
380 }
381
382
383
384 /**
385 * Indicates whether this tool defaults to launching in interactive mode if
386 * the tool is invoked without any command-line arguments. This will only be
387 * used if {@link #supportsInteractiveMode()} returns {@code true}.
388 *
389 * @return {@code true} if this tool defaults to using interactive mode if
390 * launched without any command-line arguments, or {@code false} if
391 * not.
392 */
393 @Override()
394 public boolean defaultsToInteractiveMode()
395 {
396 return true;
397 }
398
399
400
401 /**
402 * Indicates whether this tool should provide arguments for redirecting output
403 * to a file. If this method returns {@code true}, then the tool will offer
404 * an "--outputFile" argument that will specify the path to a file to which
405 * all standard output and standard error content will be written, and it will
406 * also offer a "--teeToStandardOut" argument that can only be used if the
407 * "--outputFile" argument is present and will cause all output to be written
408 * to both the specified output file and to standard output.
409 *
410 * @return {@code true} if this tool should provide arguments for redirecting
411 * output to a file, or {@code false} if not.
412 */
413 @Override()
414 protected boolean supportsOutputFile()
415 {
416 return true;
417 }
418
419
420
421 /**
422 * Indicates whether this tool should default to interactively prompting for
423 * the bind password if a password is required but no argument was provided
424 * to indicate how to get the password.
425 *
426 * @return {@code true} if this tool should default to interactively
427 * prompting for the bind password, or {@code false} if not.
428 */
429 @Override()
430 protected boolean defaultToPromptForBindPassword()
431 {
432 return true;
433 }
434
435
436
437 /**
438 * Indicates whether this tool supports the use of a properties file for
439 * specifying default values for arguments that aren't specified on the
440 * command line.
441 *
442 * @return {@code true} if this tool supports the use of a properties file
443 * for specifying default values for arguments that aren't specified
444 * on the command line, or {@code false} if not.
445 */
446 @Override()
447 public boolean supportsPropertiesFile()
448 {
449 return true;
450 }
451
452
453
454 /**
455 * Indicates whether the LDAP-specific arguments should include alternate
456 * versions of all long identifiers that consist of multiple words so that
457 * they are available in both camelCase and dash-separated versions.
458 *
459 * @return {@code true} if this tool should provide multiple versions of
460 * long identifiers for LDAP-specific arguments, or {@code false} if
461 * not.
462 */
463 @Override()
464 protected boolean includeAlternateLongIdentifiers()
465 {
466 return true;
467 }
468
469
470
471 /**
472 * Adds the arguments used by this program that aren't already provided by the
473 * generic {@code LDAPCommandLineTool} framework.
474 *
475 * @param parser The argument parser to which the arguments should be added.
476 *
477 * @throws ArgumentException If a problem occurs while adding the arguments.
478 */
479 @Override()
480 public void addNonLDAPArguments(final ArgumentParser parser)
481 throws ArgumentException
482 {
483 String description = "The base DN to use for the searches. It may be a " +
484 "simple DN or a value pattern to specify a range of DNs (e.g., " +
485 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " +
486 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
487 "value pattern syntax. This must be provided.";
488 baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
489 baseDN.setArgumentGroupName("Search And Modification Arguments");
490 baseDN.addLongIdentifier("base-dn");
491 parser.addArgument(baseDN);
492
493
494 description = "The scope to use for the searches. It should be 'base', " +
495 "'one', 'sub', or 'subord'. If this is not provided, then " +
496 "a default scope of 'sub' will be used.";
497 scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
498 SearchScope.SUB);
499 scopeArg.setArgumentGroupName("Search And Modification Arguments");
500 parser.addArgument(scopeArg);
501
502
503 description = "The filter to use for the searches. It may be a simple " +
504 "filter or a value pattern to specify a range of filters " +
505 "(e.g., \"(uid=user.[1-1000])\"). See " +
506 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details " +
507 "about the value pattern syntax. This must be provided.";
508 filter = new StringArgument('f', "filter", true, 1, "{filter}",
509 description);
510 filter.setArgumentGroupName("Search And Modification Arguments");
511 parser.addArgument(filter);
512
513
514 description = "The name of an attribute to include in entries returned " +
515 "from the searches. Multiple attributes may be requested " +
516 "by providing this argument multiple times. If no request " +
517 "attributes are provided, then the entries returned will " +
518 "include all user attributes.";
519 returnAttributes = new StringArgument('A', "attribute", false, 0, "{name}",
520 description);
521 returnAttributes.setArgumentGroupName("Search And Modification Arguments");
522 parser.addArgument(returnAttributes);
523
524
525 description = "The name of the attribute to modify. Multiple attributes " +
526 "may be specified by providing this argument multiple " +
527 "times. At least one attribute must be specified.";
528 modifyAttributes = new StringArgument('m', "modifyAttribute", true, 0,
529 "{name}", description);
530 modifyAttributes.setArgumentGroupName("Search And Modification Arguments");
531 modifyAttributes.addLongIdentifier("modify-attribute");
532 parser.addArgument(modifyAttributes);
533
534
535 description = "The length in bytes to use when generating values for the " +
536 "modifications. If this is not provided, then a default " +
537 "length of ten bytes will be used.";
538 valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}",
539 description, 1, Integer.MAX_VALUE, 10);
540 valueLength.setArgumentGroupName("Search And Modification Arguments");
541 valueLength.addLongIdentifier("value-length");
542 parser.addArgument(valueLength);
543
544
545 description = "The set of characters to use to generate the values for " +
546 "the modifications. It should only include ASCII " +
547 "characters. If this is not provided, then a default set " +
548 "of lowercase alphabetic characters will be used.";
549 characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}",
550 description,
551 "abcdefghijklmnopqrstuvwxyz");
552 characterSet.setArgumentGroupName("Search And Modification Arguments");
553 characterSet.addLongIdentifier("character-set");
554 parser.addArgument(characterSet);
555
556
557 description = "The number of threads to use to perform the searches. If " +
558 "this is not provided, then a default of one thread will " +
559 "be used.";
560 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
561 description, 1, Integer.MAX_VALUE, 1);
562 numThreads.setArgumentGroupName("Rate Management Arguments");
563 numThreads.addLongIdentifier("num-threads");
564 parser.addArgument(numThreads);
565
566
567 description = "The length of time in seconds between output lines. If " +
568 "this is not provided, then a default interval of five " +
569 "seconds will be used.";
570 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
571 "{num}", description, 1,
572 Integer.MAX_VALUE, 5);
573 collectionInterval.setArgumentGroupName("Rate Management Arguments");
574 collectionInterval.addLongIdentifier("interval-duration");
575 parser.addArgument(collectionInterval);
576
577
578 description = "The maximum number of intervals for which to run. If " +
579 "this is not provided, then the tool will run until it is " +
580 "interrupted.";
581 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
582 description, 1, Integer.MAX_VALUE,
583 Integer.MAX_VALUE);
584 numIntervals.setArgumentGroupName("Rate Management Arguments");
585 numIntervals.addLongIdentifier("num-intervals");
586 parser.addArgument(numIntervals);
587
588 description = "The number of search and modify iterations that should be " +
589 "processed on a connection before that connection is " +
590 "closed and replaced with a newly-established (and " +
591 "authenticated, if appropriate) connection. If this is " +
592 "not provided, then connections will not be periodically " +
593 "closed and re-established.";
594 iterationsBeforeReconnect = new IntegerArgument(null,
595 "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
596 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
597 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect");
598 parser.addArgument(iterationsBeforeReconnect);
599
600 description = "The target number of searches to perform per second. It " +
601 "is still necessary to specify a sufficient number of " +
602 "threads for achieving this rate. If neither this option " +
603 "nor --variableRateData is provided, then the tool will " +
604 "run at the maximum rate for the specified number of " +
605 "threads.";
606 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
607 "{searches-per-second}", description,
608 1, Integer.MAX_VALUE);
609 ratePerSecond.setArgumentGroupName("Rate Management Arguments");
610 ratePerSecond.addLongIdentifier("rate-per-second");
611 parser.addArgument(ratePerSecond);
612
613 final String variableRateDataArgName = "variableRateData";
614 final String generateSampleRateFileArgName = "generateSampleRateFile";
615 description = RateAdjustor.getVariableRateDataArgumentDescription(
616 generateSampleRateFileArgName);
617 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
618 "{path}", description, true, true, true,
619 false);
620 variableRateData.setArgumentGroupName("Rate Management Arguments");
621 variableRateData.addLongIdentifier("variable-rate-data");
622 parser.addArgument(variableRateData);
623
624 description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
625 variableRateDataArgName);
626 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
627 false, 1, "{path}", description, false,
628 true, true, false);
629 sampleRateFile.setArgumentGroupName("Rate Management Arguments");
630 sampleRateFile.addLongIdentifier("generate-sample-rate-file");
631 sampleRateFile.setUsageArgument(true);
632 parser.addArgument(sampleRateFile);
633 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
634
635 description = "The number of intervals to complete before beginning " +
636 "overall statistics collection. Specifying a nonzero " +
637 "number of warm-up intervals gives the client and server " +
638 "a chance to warm up without skewing performance results.";
639 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
640 "{num}", description, 0, Integer.MAX_VALUE, 0);
641 warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
642 warmUpIntervals.addLongIdentifier("warm-up-intervals");
643 parser.addArgument(warmUpIntervals);
644
645 description = "Indicates the format to use for timestamps included in " +
646 "the output. A value of 'none' indicates that no " +
647 "timestamps should be included. A value of 'with-date' " +
648 "indicates that both the date and the time should be " +
649 "included. A value of 'without-date' indicates that only " +
650 "the time should be included.";
651 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
652 allowedFormats.add("none");
653 allowedFormats.add("with-date");
654 allowedFormats.add("without-date");
655 timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
656 "{format}", description, allowedFormats, "none");
657 timestampFormat.addLongIdentifier("timestamp-format");
658 parser.addArgument(timestampFormat);
659
660 description = "Indicates that the proxied authorization control (as " +
661 "defined in RFC 4370) should be used to request that " +
662 "operations be processed using an alternate authorization " +
663 "identity. This may be a simple authorization ID or it " +
664 "may be a value pattern to specify a range of " +
665 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL +
666 " for complete details about the value pattern syntax.";
667 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
668 description);
669 proxyAs.addLongIdentifier("proxy-as");
670 parser.addArgument(proxyAs);
671
672 description = "Indicates that information about the result codes for " +
673 "failed operations should not be displayed.";
674 suppressErrors = new BooleanArgument(null,
675 "suppressErrorResultCodes", 1, description);
676 suppressErrors.addLongIdentifier("suppress-error-result-codes");
677 parser.addArgument(suppressErrors);
678
679 description = "Generate output in CSV format rather than a " +
680 "display-friendly format";
681 csvFormat = new BooleanArgument('c', "csv", 1, description);
682 parser.addArgument(csvFormat);
683
684 description = "Specifies the seed to use for the random number generator.";
685 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
686 description);
687 randomSeed.addLongIdentifier("random-seed");
688 parser.addArgument(randomSeed);
689 }
690
691
692
693 /**
694 * Indicates whether this tool supports creating connections to multiple
695 * servers. If it is to support multiple servers, then the "--hostname" and
696 * "--port" arguments will be allowed to be provided multiple times, and
697 * will be required to be provided the same number of times. The same type of
698 * communication security and bind credentials will be used for all servers.
699 *
700 * @return {@code true} if this tool supports creating connections to
701 * multiple servers, or {@code false} if not.
702 */
703 @Override()
704 protected boolean supportsMultipleServers()
705 {
706 return true;
707 }
708
709
710
711 /**
712 * Retrieves the connection options that should be used for connections
713 * created for use with this tool.
714 *
715 * @return The connection options that should be used for connections created
716 * for use with this tool.
717 */
718 @Override()
719 public LDAPConnectionOptions getConnectionOptions()
720 {
721 final LDAPConnectionOptions options = new LDAPConnectionOptions();
722 options.setUseSynchronousMode(true);
723 return options;
724 }
725
726
727
728 /**
729 * Performs the actual processing for this tool. In this case, it gets a
730 * connection to the directory server and uses it to perform the requested
731 * searches.
732 *
733 * @return The result code for the processing that was performed.
734 */
735 @Override()
736 public ResultCode doToolProcessing()
737 {
738 runningThread = Thread.currentThread();
739
740 try
741 {
742 return doToolProcessingInternal();
743 }
744 finally
745 {
746 runningThread = null;
747 }
748 }
749
750
751
752 /**
753 * Performs the actual processing for this tool. In this case, it gets a
754 * connection to the directory server and uses it to perform the requested
755 * searches.
756 *
757 * @return The result code for the processing that was performed.
758 */
759 private ResultCode doToolProcessingInternal()
760 {
761 // If the sample rate file argument was specified, then generate the sample
762 // variable rate data file and return.
763 if (sampleRateFile.isPresent())
764 {
765 try
766 {
767 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
768 return ResultCode.SUCCESS;
769 }
770 catch (final Exception e)
771 {
772 debugException(e);
773 err("An error occurred while trying to write sample variable data " +
774 "rate file '", sampleRateFile.getValue().getAbsolutePath(),
775 "': ", getExceptionMessage(e));
776 return ResultCode.LOCAL_ERROR;
777 }
778 }
779
780
781 // Determine the random seed to use.
782 final Long seed;
783 if (randomSeed.isPresent())
784 {
785 seed = Long.valueOf(randomSeed.getValue());
786 }
787 else
788 {
789 seed = null;
790 }
791
792 // Create value patterns for the base DN, filter, and proxied authorization
793 // DN.
794 final ValuePattern dnPattern;
795 try
796 {
797 dnPattern = new ValuePattern(baseDN.getValue(), seed);
798 }
799 catch (final ParseException pe)
800 {
801 debugException(pe);
802 err("Unable to parse the base DN value pattern: ", pe.getMessage());
803 return ResultCode.PARAM_ERROR;
804 }
805
806 final ValuePattern filterPattern;
807 try
808 {
809 filterPattern = new ValuePattern(filter.getValue(), seed);
810 }
811 catch (final ParseException pe)
812 {
813 debugException(pe);
814 err("Unable to parse the filter pattern: ", pe.getMessage());
815 return ResultCode.PARAM_ERROR;
816 }
817
818 final ValuePattern authzIDPattern;
819 if (proxyAs.isPresent())
820 {
821 try
822 {
823 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
824 }
825 catch (final ParseException pe)
826 {
827 debugException(pe);
828 err("Unable to parse the proxied authorization pattern: ",
829 pe.getMessage());
830 return ResultCode.PARAM_ERROR;
831 }
832 }
833 else
834 {
835 authzIDPattern = null;
836 }
837
838
839 // Get the attributes to return.
840 final String[] returnAttrs;
841 if (returnAttributes.isPresent())
842 {
843 final List<String> attrList = returnAttributes.getValues();
844 returnAttrs = new String[attrList.size()];
845 attrList.toArray(returnAttrs);
846 }
847 else
848 {
849 returnAttrs = NO_STRINGS;
850 }
851
852
853 // Get the names of the attributes to modify.
854 final String[] modAttrs = new String[modifyAttributes.getValues().size()];
855 modifyAttributes.getValues().toArray(modAttrs);
856
857
858 // Get the character set as a byte array.
859 final byte[] charSet = getBytes(characterSet.getValue());
860
861
862 // If the --ratePerSecond option was specified, then limit the rate
863 // accordingly.
864 FixedRateBarrier fixedRateBarrier = null;
865 if (ratePerSecond.isPresent() || variableRateData.isPresent())
866 {
867 // We might not have a rate per second if --variableRateData is specified.
868 // The rate typically doesn't matter except when we have warm-up
869 // intervals. In this case, we'll run at the max rate.
870 final int intervalSeconds = collectionInterval.getValue();
871 final int ratePerInterval =
872 (ratePerSecond.getValue() == null)
873 ? Integer.MAX_VALUE
874 : ratePerSecond.getValue() * intervalSeconds;
875 fixedRateBarrier =
876 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
877 }
878
879
880 // If --variableRateData was specified, then initialize a RateAdjustor.
881 RateAdjustor rateAdjustor = null;
882 if (variableRateData.isPresent())
883 {
884 try
885 {
886 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
887 ratePerSecond.getValue(), variableRateData.getValue());
888 }
889 catch (final IOException e)
890 {
891 debugException(e);
892 err("Initializing the variable rates failed: " + e.getMessage());
893 return ResultCode.PARAM_ERROR;
894 }
895 catch (final IllegalArgumentException e)
896 {
897 debugException(e);
898 err("Initializing the variable rates failed: " + e.getMessage());
899 return ResultCode.PARAM_ERROR;
900 }
901 }
902
903
904 // Determine whether to include timestamps in the output and if so what
905 // format should be used for them.
906 final boolean includeTimestamp;
907 final String timeFormat;
908 if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
909 {
910 includeTimestamp = true;
911 timeFormat = "dd/MM/yyyy HH:mm:ss";
912 }
913 else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
914 {
915 includeTimestamp = true;
916 timeFormat = "HH:mm:ss";
917 }
918 else
919 {
920 includeTimestamp = false;
921 timeFormat = null;
922 }
923
924
925 // Determine whether any warm-up intervals should be run.
926 final long totalIntervals;
927 final boolean warmUp;
928 int remainingWarmUpIntervals = warmUpIntervals.getValue();
929 if (remainingWarmUpIntervals > 0)
930 {
931 warmUp = true;
932 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
933 }
934 else
935 {
936 warmUp = true;
937 totalIntervals = 0L + numIntervals.getValue();
938 }
939
940
941 // Create the table that will be used to format the output.
942 final OutputFormat outputFormat;
943 if (csvFormat.isPresent())
944 {
945 outputFormat = OutputFormat.CSV;
946 }
947 else
948 {
949 outputFormat = OutputFormat.COLUMNS;
950 }
951
952 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
953 timeFormat, outputFormat, " ",
954 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
955 "Searches/Sec"),
956 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
957 "Srch Dur ms"),
958 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
959 "Mods/Sec"),
960 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
961 "Mod Dur ms"),
962 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
963 "Errors/Sec"),
964 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
965 "Searches/Sec"),
966 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
967 "Srch Dur ms"),
968 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
969 "Mods/Sec"),
970 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
971 "Mod Dur ms"));
972
973
974 // Create values to use for statistics collection.
975 final AtomicLong searchCounter = new AtomicLong(0L);
976 final AtomicLong errorCounter = new AtomicLong(0L);
977 final AtomicLong modCounter = new AtomicLong(0L);
978 final AtomicLong modDurations = new AtomicLong(0L);
979 final AtomicLong searchDurations = new AtomicLong(0L);
980 final ResultCodeCounter rcCounter = new ResultCodeCounter();
981
982
983 // Determine the length of each interval in milliseconds.
984 final long intervalMillis = 1000L * collectionInterval.getValue();
985
986
987 // Create the threads to use for the searches.
988 final Random random = new Random();
989 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
990 final SearchAndModRateThread[] threads =
991 new SearchAndModRateThread[numThreads.getValue()];
992 for (int i=0; i < threads.length; i++)
993 {
994 final LDAPConnection connection;
995 try
996 {
997 connection = getConnection();
998 }
999 catch (final LDAPException le)
1000 {
1001 debugException(le);
1002 err("Unable to connect to the directory server: ",
1003 getExceptionMessage(le));
1004 return le.getResultCode();
1005 }
1006
1007 threads[i] = new SearchAndModRateThread(this, i, connection, dnPattern,
1008 scopeArg.getValue(), filterPattern, returnAttrs, modAttrs,
1009 valueLength.getValue(), charSet, authzIDPattern,
1010 iterationsBeforeReconnect.getValue(), random.nextLong(), barrier,
1011 searchCounter, modCounter, searchDurations, modDurations,
1012 errorCounter, rcCounter, fixedRateBarrier);
1013 threads[i].start();
1014 }
1015
1016
1017 // Display the table header.
1018 for (final String headerLine : formatter.getHeaderLines(true))
1019 {
1020 out(headerLine);
1021 }
1022
1023
1024 // Start the RateAdjustor before the threads so that the initial value is
1025 // in place before any load is generated unless we're doing a warm-up in
1026 // which case, we'll start it after the warm-up is complete.
1027 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1028 {
1029 rateAdjustor.start();
1030 }
1031
1032
1033 // Indicate that the threads can start running.
1034 try
1035 {
1036 barrier.await();
1037 }
1038 catch (final Exception e)
1039 {
1040 debugException(e);
1041 }
1042
1043 long overallStartTime = System.nanoTime();
1044 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1045
1046
1047 boolean setOverallStartTime = false;
1048 long lastSearchDuration = 0L;
1049 long lastModDuration = 0L;
1050 long lastNumErrors = 0L;
1051 long lastNumSearches = 0L;
1052 long lastNumMods = 0L;
1053 long lastEndTime = System.nanoTime();
1054 for (long i=0; i < totalIntervals; i++)
1055 {
1056 if (rateAdjustor != null)
1057 {
1058 if (! rateAdjustor.isAlive())
1059 {
1060 out("All of the rates in " + variableRateData.getValue().getName() +
1061 " have been completed.");
1062 break;
1063 }
1064 }
1065
1066 final long startTimeMillis = System.currentTimeMillis();
1067 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1068 nextIntervalStartTime += intervalMillis;
1069 if (sleepTimeMillis > 0)
1070 {
1071 sleeper.sleep(sleepTimeMillis);
1072 }
1073
1074 if (stopRequested.get())
1075 {
1076 break;
1077 }
1078
1079 final long endTime = System.nanoTime();
1080 final long intervalDuration = endTime - lastEndTime;
1081
1082 final long numSearches;
1083 final long numMods;
1084 final long numErrors;
1085 final long totalSearchDuration;
1086 final long totalModDuration;
1087 if (warmUp && (remainingWarmUpIntervals > 0))
1088 {
1089 numSearches = searchCounter.getAndSet(0L);
1090 numMods = modCounter.getAndSet(0L);
1091 numErrors = errorCounter.getAndSet(0L);
1092 totalSearchDuration = searchDurations.getAndSet(0L);
1093 totalModDuration = modDurations.getAndSet(0L);
1094 }
1095 else
1096 {
1097 numSearches = searchCounter.get();
1098 numMods = modCounter.get();
1099 numErrors = errorCounter.get();
1100 totalSearchDuration = searchDurations.get();
1101 totalModDuration = modDurations.get();
1102 }
1103
1104 final long recentNumSearches = numSearches - lastNumSearches;
1105 final long recentNumMods = numMods - lastNumMods;
1106 final long recentNumErrors = numErrors - lastNumErrors;
1107 final long recentSearchDuration =
1108 totalSearchDuration - lastSearchDuration;
1109 final long recentModDuration = totalModDuration - lastModDuration;
1110
1111 final double numSeconds = intervalDuration / 1000000000.0d;
1112 final double recentSearchRate = recentNumSearches / numSeconds;
1113 final double recentModRate = recentNumMods / numSeconds;
1114 final double recentErrorRate = recentNumErrors / numSeconds;
1115
1116 final double recentAvgSearchDuration;
1117 if (recentNumSearches > 0L)
1118 {
1119 recentAvgSearchDuration =
1120 1.0d * recentSearchDuration / recentNumSearches / 1000000;
1121 }
1122 else
1123 {
1124 recentAvgSearchDuration = 0.0d;
1125 }
1126
1127 final double recentAvgModDuration;
1128 if (recentNumMods > 0L)
1129 {
1130 recentAvgModDuration =
1131 1.0d * recentModDuration / recentNumMods / 1000000;
1132 }
1133 else
1134 {
1135 recentAvgModDuration = 0.0d;
1136 }
1137
1138 if (warmUp && (remainingWarmUpIntervals > 0))
1139 {
1140 out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1141 recentModRate, recentAvgModDuration, recentErrorRate, "warming up",
1142 "warming up", "warming up", "warming up"));
1143
1144 remainingWarmUpIntervals--;
1145 if (remainingWarmUpIntervals == 0)
1146 {
1147 out("Warm-up completed. Beginning overall statistics collection.");
1148 setOverallStartTime = true;
1149 if (rateAdjustor != null)
1150 {
1151 rateAdjustor.start();
1152 }
1153 }
1154 }
1155 else
1156 {
1157 if (setOverallStartTime)
1158 {
1159 overallStartTime = lastEndTime;
1160 setOverallStartTime = false;
1161 }
1162
1163 final double numOverallSeconds =
1164 (endTime - overallStartTime) / 1000000000.0d;
1165 final double overallSearchRate = numSearches / numOverallSeconds;
1166 final double overallModRate = numMods / numOverallSeconds;
1167
1168 final double overallAvgSearchDuration;
1169 if (numSearches > 0L)
1170 {
1171 overallAvgSearchDuration =
1172 1.0d * totalSearchDuration / numSearches / 1000000;
1173 }
1174 else
1175 {
1176 overallAvgSearchDuration = 0.0d;
1177 }
1178
1179 final double overallAvgModDuration;
1180 if (numMods > 0L)
1181 {
1182 overallAvgModDuration =
1183 1.0d * totalModDuration / numMods / 1000000;
1184 }
1185 else
1186 {
1187 overallAvgModDuration = 0.0d;
1188 }
1189
1190 out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1191 recentModRate, recentAvgModDuration, recentErrorRate,
1192 overallSearchRate, overallAvgSearchDuration, overallModRate,
1193 overallAvgModDuration));
1194
1195 lastNumSearches = numSearches;
1196 lastNumMods = numMods;
1197 lastNumErrors = numErrors;
1198 lastSearchDuration = totalSearchDuration;
1199 lastModDuration = totalModDuration;
1200 }
1201
1202 final List<ObjectPair<ResultCode,Long>> rcCounts =
1203 rcCounter.getCounts(true);
1204 if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1205 {
1206 err("\tError Results:");
1207 for (final ObjectPair<ResultCode,Long> p : rcCounts)
1208 {
1209 err("\t", p.getFirst().getName(), ": ", p.getSecond());
1210 }
1211 }
1212
1213 lastEndTime = endTime;
1214 }
1215
1216
1217 // Shut down the RateAdjustor if we have one.
1218 if (rateAdjustor != null)
1219 {
1220 rateAdjustor.shutDown();
1221 }
1222
1223 // Stop all of the threads.
1224 ResultCode resultCode = ResultCode.SUCCESS;
1225 for (final SearchAndModRateThread t : threads)
1226 {
1227 final ResultCode r = t.stopRunning();
1228 if (resultCode == ResultCode.SUCCESS)
1229 {
1230 resultCode = r;
1231 }
1232 }
1233
1234 return resultCode;
1235 }
1236
1237
1238
1239 /**
1240 * Requests that this tool stop running. This method will attempt to wait
1241 * for all threads to complete before returning control to the caller.
1242 */
1243 public void stopRunning()
1244 {
1245 stopRequested.set(true);
1246 sleeper.wakeup();
1247
1248 final Thread t = runningThread;
1249 if (t != null)
1250 {
1251 try
1252 {
1253 t.join();
1254 }
1255 catch (final Exception e)
1256 {
1257 debugException(e);
1258 }
1259 }
1260 }
1261
1262
1263
1264 /**
1265 * {@inheritDoc}
1266 */
1267 @Override()
1268 public LinkedHashMap<String[],String> getExampleUsages()
1269 {
1270 final LinkedHashMap<String[],String> examples =
1271 new LinkedHashMap<String[],String>(2);
1272
1273 String[] args =
1274 {
1275 "--hostname", "server.example.com",
1276 "--port", "389",
1277 "--bindDN", "uid=admin,dc=example,dc=com",
1278 "--bindPassword", "password",
1279 "--baseDN", "dc=example,dc=com",
1280 "--scope", "sub",
1281 "--filter", "(uid=user.[1-1000000])",
1282 "--attribute", "givenName",
1283 "--attribute", "sn",
1284 "--attribute", "mail",
1285 "--modifyAttribute", "description",
1286 "--valueLength", "10",
1287 "--characterSet", "abcdefghijklmnopqrstuvwxyz0123456789",
1288 "--numThreads", "10"
1289 };
1290 String description =
1291 "Test search and modify performance by searching randomly across a " +
1292 "set of one million users located below 'dc=example,dc=com' with " +
1293 "ten concurrent threads. The entries returned to the client will " +
1294 "include the givenName, sn, and mail attributes, and the " +
1295 "description attribute of each entry returned will be replaced " +
1296 "with a string of ten randomly-selected alphanumeric characters.";
1297 examples.put(args, description);
1298
1299 args = new String[]
1300 {
1301 "--generateSampleRateFile", "variable-rate-data.txt"
1302 };
1303 description =
1304 "Generate a sample variable rate definition file that may be used " +
1305 "in conjunction with the --variableRateData argument. The sample " +
1306 "file will include comments that describe the format for data to be " +
1307 "included in this file.";
1308 examples.put(args, description);
1309
1310 return examples;
1311 }
1312 }