001 /*
002 * Copyright 2008-2016 UnboundID Corp.
003 * All Rights Reserved.
004 */
005 /*
006 * Copyright (C) 2008-2016 UnboundID Corp.
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021 package com.unboundid.ldap.sdk.examples;
022
023
024
025 import java.io.IOException;
026 import java.io.OutputStream;
027 import java.io.Serializable;
028 import java.text.ParseException;
029 import java.util.ArrayList;
030 import java.util.LinkedHashMap;
031 import java.util.LinkedHashSet;
032 import java.util.List;
033 import java.util.Random;
034 import java.util.concurrent.CyclicBarrier;
035 import java.util.concurrent.atomic.AtomicBoolean;
036 import java.util.concurrent.atomic.AtomicLong;
037
038 import com.unboundid.ldap.sdk.Control;
039 import com.unboundid.ldap.sdk.LDAPConnection;
040 import com.unboundid.ldap.sdk.LDAPConnectionOptions;
041 import com.unboundid.ldap.sdk.LDAPException;
042 import com.unboundid.ldap.sdk.ResultCode;
043 import com.unboundid.ldap.sdk.Version;
044 import com.unboundid.ldap.sdk.controls.AssertionRequestControl;
045 import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl;
046 import com.unboundid.ldap.sdk.controls.PostReadRequestControl;
047 import com.unboundid.ldap.sdk.controls.PreReadRequestControl;
048 import com.unboundid.util.ColumnFormatter;
049 import com.unboundid.util.FixedRateBarrier;
050 import com.unboundid.util.FormattableColumn;
051 import com.unboundid.util.HorizontalAlignment;
052 import com.unboundid.util.LDAPCommandLineTool;
053 import com.unboundid.util.ObjectPair;
054 import com.unboundid.util.OutputFormat;
055 import com.unboundid.util.RateAdjustor;
056 import com.unboundid.util.ResultCodeCounter;
057 import com.unboundid.util.ThreadSafety;
058 import com.unboundid.util.ThreadSafetyLevel;
059 import com.unboundid.util.ValuePattern;
060 import com.unboundid.util.WakeableSleeper;
061 import com.unboundid.util.args.ArgumentException;
062 import com.unboundid.util.args.ArgumentParser;
063 import com.unboundid.util.args.BooleanArgument;
064 import com.unboundid.util.args.ControlArgument;
065 import com.unboundid.util.args.FileArgument;
066 import com.unboundid.util.args.FilterArgument;
067 import com.unboundid.util.args.IntegerArgument;
068 import com.unboundid.util.args.StringArgument;
069
070 import static com.unboundid.util.Debug.*;
071 import static com.unboundid.util.StaticUtils.*;
072
073
074
075 /**
076 * This class provides a tool that can be used to perform repeated modifications
077 * in an LDAP directory server using multiple threads. It can help provide an
078 * estimate of the modify performance that a directory server is able to
079 * achieve. The target entry DN may be a value pattern as described in the
080 * {@link ValuePattern} class. This makes it possible to modify a range of
081 * entries rather than repeatedly updating the same entry.
082 * <BR><BR>
083 * Some of the APIs demonstrated by this example include:
084 * <UL>
085 * <LI>Argument Parsing (from the {@code com.unboundid.util.args}
086 * package)</LI>
087 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
088 * package)</LI>
089 * <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
090 * package)</LI>
091 * <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
092 * </UL>
093 * <BR><BR>
094 * All of the necessary information is provided using command line arguments.
095 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
096 * class, as well as the following additional arguments:
097 * <UL>
098 * <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the
099 * entry to be modified. This must be provided. It may be a simple DN,
100 * or it may be a value pattern to express a range of entry DNs.</LI>
101 * <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the
102 * attribute to modify. Multiple attributes may be modified by providing
103 * multiple instances of this argument. At least one attribute must be
104 * provided.</LI>
105 * <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
106 * use for the values of the target attributes. If this is not provided,
107 * then a default length of 10 bytes will be used.</LI>
108 * <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
109 * characters that will be used to generate the values to use for the
110 * target attributes. It should only include ASCII characters. Values
111 * will be generated from randomly-selected characters from this set. If
112 * this is not provided, then a default set of lowercase alphabetic
113 * characters will be used.</LI>
114 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
115 * concurrent threads to use when performing the modifications. If this
116 * is not provided, then a default of one thread will be used.</LI>
117 * <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
118 * time in seconds between lines out output. If this is not provided,
119 * then a default interval duration of five seconds will be used.</LI>
120 * <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
121 * intervals for which to run. If this is not provided, then it will
122 * run forever.</LI>
123 * <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify
124 * iterations that should be performed on a connection before that
125 * connection is closed and replaced with a newly-established (and
126 * authenticated, if appropriate) connection.</LI>
127 * <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}"
128 * -- specifies the target number of modifies to perform per second. It
129 * is still necessary to specify a sufficient number of threads for
130 * achieving this rate. If this option is not provided, then the tool
131 * will run at the maximum rate for the specified number of threads.</LI>
132 * <LI>"--variableRateData {path}" -- specifies the path to a file containing
133 * information needed to allow the tool to vary the target rate over time.
134 * If this option is not provided, then the tool will either use a fixed
135 * target rate as specified by the "--ratePerSecond" argument, or it will
136 * run at the maximum rate.</LI>
137 * <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
138 * which sample data will be written illustrating and describing the
139 * format of the file expected to be used in conjunction with the
140 * "--variableRateData" argument.</LI>
141 * <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
142 * complete before beginning overall statistics collection.</LI>
143 * <LI>"--timestampFormat {format}" -- specifies the format to use for
144 * timestamps included before each output line. The format may be one of
145 * "none" (for no timestamps), "with-date" (to include both the date and
146 * the time), or "without-date" (to include only time time).</LI>
147 * <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
148 * authorization v2 control to request that the operation be processed
149 * using an alternate authorization identity. In this case, the bind DN
150 * should be that of a user that has permission to use this control. The
151 * authorization identity may be a value pattern.</LI>
152 * <LI>"--suppressErrorResultCodes" -- Indicates that information about the
153 * result codes for failed operations should not be displayed.</LI>
154 * <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
155 * display-friendly format.</LI>
156 * </UL>
157 */
158 @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
159 public final class ModRate
160 extends LDAPCommandLineTool
161 implements Serializable
162 {
163 /**
164 * The serial version UID for this serializable class.
165 */
166 private static final long serialVersionUID = 2709717414202815822L;
167
168
169
170 // Indicates whether a request has been made to stop running.
171 private final AtomicBoolean stopRequested;
172
173 // The argument used to indicate whether to generate output in CSV format.
174 private BooleanArgument csvFormat;
175
176 // Indicates that the tool should use the increment modification type instead
177 // of replace.
178 private BooleanArgument increment;
179
180 // Indicates that modify requests should include the permissive modify request
181 // control.
182 private BooleanArgument permissiveModify;
183
184 // The argument used to indicate whether to suppress information about error
185 // result codes.
186 private BooleanArgument suppressErrorsArgument;
187
188 // The argument used to indicate that a generic control should be included in
189 // the request.
190 private ControlArgument control;
191
192 // The argument used to specify a variable rate file.
193 private FileArgument sampleRateFile;
194
195 // The argument used to specify a variable rate file.
196 private FileArgument variableRateData;
197
198 // Indicates that modify requests should include the assertion request control
199 // with the specified filter.
200 private FilterArgument assertionFilter;
201
202 // The argument used to specify the collection interval.
203 private IntegerArgument collectionInterval;
204
205 // The increment amount to use when performing an increment instead of a
206 // replace.
207 private IntegerArgument incrementAmount;
208
209 // The argument used to specify the number of modify iterations on a
210 // connection before it is closed and re-established.
211 private IntegerArgument iterationsBeforeReconnect;
212
213 // The argument used to specify the number of intervals.
214 private IntegerArgument numIntervals;
215
216 // The argument used to specify the number of threads.
217 private IntegerArgument numThreads;
218
219 // The argument used to specify the seed to use for the random number
220 // generator.
221 private IntegerArgument randomSeed;
222
223 // The target rate of modifies per second.
224 private IntegerArgument ratePerSecond;
225
226 // The number of values to include in the replace modification.
227 private IntegerArgument valueCount;
228
229 // The argument used to specify the length of the values to generate.
230 private IntegerArgument valueLength;
231
232 // The number of warm-up intervals to perform.
233 private IntegerArgument warmUpIntervals;
234
235 // The argument used to specify the name of the attribute to modify.
236 private StringArgument attribute;
237
238 // The argument used to specify the set of characters to use when generating
239 // values.
240 private StringArgument characterSet;
241
242 // The argument used to specify the DNs of the entries to modify.
243 private StringArgument entryDN;
244
245 // Indicates that modify requests should include the post-read request control
246 // to request the specified attribute.
247 private StringArgument postReadAttribute;
248
249 // Indicates that modify requests should include the pre-read request control
250 // to request the specified attribute.
251 private StringArgument preReadAttribute;
252
253 // The argument used to specify the proxied authorization identity.
254 private StringArgument proxyAs;
255
256 // The argument used to specify the timestamp format.
257 private StringArgument timestampFormat;
258
259 // The thread currently being used to run the searchrate tool.
260 private volatile Thread runningThread;
261
262 // A wakeable sleeper that will be used to sleep between reporting intervals.
263 private final WakeableSleeper sleeper;
264
265
266
267 /**
268 * Parse the provided command line arguments and make the appropriate set of
269 * changes.
270 *
271 * @param args The command line arguments provided to this program.
272 */
273 public static void main(final String[] args)
274 {
275 final ResultCode resultCode = main(args, System.out, System.err);
276 if (resultCode != ResultCode.SUCCESS)
277 {
278 System.exit(resultCode.intValue());
279 }
280 }
281
282
283
284 /**
285 * Parse the provided command line arguments and make the appropriate set of
286 * changes.
287 *
288 * @param args The command line arguments provided to this program.
289 * @param outStream The output stream to which standard out should be
290 * written. It may be {@code null} if output should be
291 * suppressed.
292 * @param errStream The output stream to which standard error should be
293 * written. It may be {@code null} if error messages
294 * should be suppressed.
295 *
296 * @return A result code indicating whether the processing was successful.
297 */
298 public static ResultCode main(final String[] args,
299 final OutputStream outStream,
300 final OutputStream errStream)
301 {
302 final ModRate modRate = new ModRate(outStream, errStream);
303 return modRate.runTool(args);
304 }
305
306
307
308 /**
309 * Creates a new instance of this tool.
310 *
311 * @param outStream The output stream to which standard out should be
312 * written. It may be {@code null} if output should be
313 * suppressed.
314 * @param errStream The output stream to which standard error should be
315 * written. It may be {@code null} if error messages
316 * should be suppressed.
317 */
318 public ModRate(final OutputStream outStream, final OutputStream errStream)
319 {
320 super(outStream, errStream);
321
322 stopRequested = new AtomicBoolean(false);
323 sleeper = new WakeableSleeper();
324 }
325
326
327
328 /**
329 * Retrieves the name for this tool.
330 *
331 * @return The name for this tool.
332 */
333 @Override()
334 public String getToolName()
335 {
336 return "modrate";
337 }
338
339
340
341 /**
342 * Retrieves the description for this tool.
343 *
344 * @return The description for this tool.
345 */
346 @Override()
347 public String getToolDescription()
348 {
349 return "Perform repeated modifications against " +
350 "an LDAP directory server.";
351 }
352
353
354
355 /**
356 * Retrieves the version string for this tool.
357 *
358 * @return The version string for this tool.
359 */
360 @Override()
361 public String getToolVersion()
362 {
363 return Version.NUMERIC_VERSION_STRING;
364 }
365
366
367
368 /**
369 * Indicates whether this tool should provide support for an interactive mode,
370 * in which the tool offers a mode in which the arguments can be provided in
371 * a text-driven menu rather than requiring them to be given on the command
372 * line. If interactive mode is supported, it may be invoked using the
373 * "--interactive" argument. Alternately, if interactive mode is supported
374 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
375 * interactive mode may be invoked by simply launching the tool without any
376 * arguments.
377 *
378 * @return {@code true} if this tool supports interactive mode, or
379 * {@code false} if not.
380 */
381 @Override()
382 public boolean supportsInteractiveMode()
383 {
384 return true;
385 }
386
387
388
389 /**
390 * Indicates whether this tool defaults to launching in interactive mode if
391 * the tool is invoked without any command-line arguments. This will only be
392 * used if {@link #supportsInteractiveMode()} returns {@code true}.
393 *
394 * @return {@code true} if this tool defaults to using interactive mode if
395 * launched without any command-line arguments, or {@code false} if
396 * not.
397 */
398 @Override()
399 public boolean defaultsToInteractiveMode()
400 {
401 return true;
402 }
403
404
405
406 /**
407 * Indicates whether this tool should provide arguments for redirecting output
408 * to a file. If this method returns {@code true}, then the tool will offer
409 * an "--outputFile" argument that will specify the path to a file to which
410 * all standard output and standard error content will be written, and it will
411 * also offer a "--teeToStandardOut" argument that can only be used if the
412 * "--outputFile" argument is present and will cause all output to be written
413 * to both the specified output file and to standard output.
414 *
415 * @return {@code true} if this tool should provide arguments for redirecting
416 * output to a file, or {@code false} if not.
417 */
418 @Override()
419 protected boolean supportsOutputFile()
420 {
421 return true;
422 }
423
424
425
426 /**
427 * Indicates whether this tool should default to interactively prompting for
428 * the bind password if a password is required but no argument was provided
429 * to indicate how to get the password.
430 *
431 * @return {@code true} if this tool should default to interactively
432 * prompting for the bind password, or {@code false} if not.
433 */
434 @Override()
435 protected boolean defaultToPromptForBindPassword()
436 {
437 return true;
438 }
439
440
441
442 /**
443 * Indicates whether this tool supports the use of a properties file for
444 * specifying default values for arguments that aren't specified on the
445 * command line.
446 *
447 * @return {@code true} if this tool supports the use of a properties file
448 * for specifying default values for arguments that aren't specified
449 * on the command line, or {@code false} if not.
450 */
451 @Override()
452 public boolean supportsPropertiesFile()
453 {
454 return true;
455 }
456
457
458
459 /**
460 * Indicates whether the LDAP-specific arguments should include alternate
461 * versions of all long identifiers that consist of multiple words so that
462 * they are available in both camelCase and dash-separated versions.
463 *
464 * @return {@code true} if this tool should provide multiple versions of
465 * long identifiers for LDAP-specific arguments, or {@code false} if
466 * not.
467 */
468 @Override()
469 protected boolean includeAlternateLongIdentifiers()
470 {
471 return true;
472 }
473
474
475
476 /**
477 * Adds the arguments used by this program that aren't already provided by the
478 * generic {@code LDAPCommandLineTool} framework.
479 *
480 * @param parser The argument parser to which the arguments should be added.
481 *
482 * @throws ArgumentException If a problem occurs while adding the arguments.
483 */
484 @Override()
485 public void addNonLDAPArguments(final ArgumentParser parser)
486 throws ArgumentException
487 {
488 String description = "The DN of the entry to modify. It may be a simple " +
489 "DN or a value pattern to specify a range of DN (e.g., " +
490 "\"uid=user.[1-1000],ou=People,dc=example,dc=com\"). See " +
491 ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
492 "value pattern syntax. This must be provided.";
493 entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description);
494 entryDN.setArgumentGroupName("Modification Arguments");
495 entryDN.addLongIdentifier("entry-dn");
496 parser.addArgument(entryDN);
497
498
499 description = "The name of the attribute to modify. Multiple attributes " +
500 "may be specified by providing this argument multiple " +
501 "times. At least one attribute must be specified.";
502 attribute = new StringArgument('A', "attribute", true, 0, "{name}",
503 description);
504 attribute.setArgumentGroupName("Modification Arguments");
505 parser.addArgument(attribute);
506
507
508 description = "The length in bytes to use when generating values for the " +
509 "replace modifications. If this is not provided, then a " +
510 "default length of ten bytes will be used.";
511 valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}",
512 description, 1, Integer.MAX_VALUE, 10);
513 valueLength.setArgumentGroupName("Modification Arguments");
514 valueLength.addLongIdentifier("value-length");
515 parser.addArgument(valueLength);
516
517
518 description = "The number of values to include in replace " +
519 "modifications. If this is not provided, then a default " +
520 "of one value will be used.";
521 valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}",
522 description, 0, Integer.MAX_VALUE, 1);
523 valueCount.setArgumentGroupName("Modification Arguments");
524 valueCount.addLongIdentifier("value-count");
525 parser.addArgument(valueCount);
526
527
528 description = "Indicates that the tool should use the increment " +
529 "modification type rather than the replace modification " +
530 "type.";
531 increment = new BooleanArgument(null, "increment", 1, description);
532 increment.setArgumentGroupName("Modification Arguments");
533 parser.addArgument(increment);
534
535
536 description = "The amount by which to increment values when using the " +
537 "increment modification type. The amount may be negative " +
538 "if values should be decremented rather than incremented. " +
539 "If this is not provided, then a default increment amount " +
540 "of one will be used.";
541 incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1,
542 null, description, Integer.MIN_VALUE,
543 Integer.MAX_VALUE, 1);
544 incrementAmount.setArgumentGroupName("Modification Arguments");
545 incrementAmount.addLongIdentifier("increment-amount");
546 parser.addArgument(incrementAmount);
547
548
549 description = "The set of characters to use to generate the values for " +
550 "the modifications. It should only include ASCII " +
551 "characters. If this is not provided, then a default set " +
552 "of lowercase alphabetic characters will be used.";
553 characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}",
554 description,
555 "abcdefghijklmnopqrstuvwxyz");
556 characterSet.setArgumentGroupName("Modification Arguments");
557 characterSet.addLongIdentifier("character-set");
558 parser.addArgument(characterSet);
559
560
561 description = "Indicates that modify requests should include the " +
562 "assertion request control with the specified filter.";
563 assertionFilter = new FilterArgument(null, "assertionFilter", false, 1,
564 "{filter}", description);
565 assertionFilter.setArgumentGroupName("Request Control Arguments");
566 assertionFilter.addLongIdentifier("assertion-filter");
567 parser.addArgument(assertionFilter);
568
569
570 description = "Indicates that modify requests should include the " +
571 "permissive modify request control.";
572 permissiveModify = new BooleanArgument(null, "permissiveModify", 1,
573 description);
574 permissiveModify.setArgumentGroupName("Request Control Arguments");
575 permissiveModify.addLongIdentifier("permissive-modify");
576 parser.addArgument(permissiveModify);
577
578
579 description = "Indicates that modify requests should include the " +
580 "pre-read request control with the specified requested " +
581 "attribute. This argument may be provided multiple times " +
582 "to indicate that multiple requested attributes should be " +
583 "included in the pre-read request control.";
584 preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0,
585 "{attribute}", description);
586 preReadAttribute.setArgumentGroupName("Request Control Arguments");
587 preReadAttribute.addLongIdentifier("pre-read-attribute");
588 parser.addArgument(preReadAttribute);
589
590
591 description = "Indicates that modify requests should include the " +
592 "post-read request control with the specified requested " +
593 "attribute. This argument may be provided multiple times " +
594 "to indicate that multiple requested attributes should be " +
595 "included in the post-read request control.";
596 postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0,
597 "{attribute}", description);
598 postReadAttribute.setArgumentGroupName("Request Control Arguments");
599 postReadAttribute.addLongIdentifier("post-read-attribute");
600 parser.addArgument(postReadAttribute);
601
602
603 description = "Indicates that the proxied authorization control (as " +
604 "defined in RFC 4370) should be used to request that " +
605 "operations be processed using an alternate authorization " +
606 "identity. This may be a simple authorization ID or it " +
607 "may be a value pattern to specify a range of " +
608 "identities. See " + ValuePattern.PUBLIC_JAVADOC_URL +
609 " for complete details about the value pattern syntax.";
610 proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
611 description);
612 proxyAs.setArgumentGroupName("Request Control Arguments");
613 proxyAs.addLongIdentifier("proxy-as");
614 parser.addArgument(proxyAs);
615
616
617 description = "Indicates that modify requests should include the " +
618 "specified request control. This may be provided multiple " +
619 "times to request multiple request controls.";
620 control = new ControlArgument('J', "control", false, 0, null, description);
621 control.setArgumentGroupName("Request Control Arguments");
622 parser.addArgument(control);
623
624
625 description = "The number of threads to use to perform the " +
626 "modifications. If this is not provided, a single thread " +
627 "will be used.";
628 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
629 description, 1, Integer.MAX_VALUE, 1);
630 numThreads.setArgumentGroupName("Rate Management Arguments");
631 numThreads.addLongIdentifier("num-threads");
632 parser.addArgument(numThreads);
633
634
635 description = "The length of time in seconds between output lines. If " +
636 "this is not provided, then a default interval of five " +
637 "seconds will be used.";
638 collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
639 "{num}", description, 1,
640 Integer.MAX_VALUE, 5);
641 collectionInterval.setArgumentGroupName("Rate Management Arguments");
642 collectionInterval.addLongIdentifier("interval-duration");
643 parser.addArgument(collectionInterval);
644
645
646 description = "The maximum number of intervals for which to run. If " +
647 "this is not provided, then the tool will run until it is " +
648 "interrupted.";
649 numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
650 description, 1, Integer.MAX_VALUE,
651 Integer.MAX_VALUE);
652 numIntervals.setArgumentGroupName("Rate Management Arguments");
653 numIntervals.addLongIdentifier("num-intervals");
654 parser.addArgument(numIntervals);
655
656 description = "The number of modify iterations that should be processed " +
657 "on a connection before that connection is closed and " +
658 "replaced with a newly-established (and authenticated, if " +
659 "appropriate) connection. If this is not provided, then " +
660 "connections will not be periodically closed and " +
661 "re-established.";
662 iterationsBeforeReconnect = new IntegerArgument(null,
663 "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
664 iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
665 iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect");
666 parser.addArgument(iterationsBeforeReconnect);
667
668 description = "The target number of modifies to perform per second. It " +
669 "is still necessary to specify a sufficient number of " +
670 "threads for achieving this rate. If neither this option " +
671 "nor --variableRateData is provided, then the tool will " +
672 "run at the maximum rate for the specified number of " +
673 "threads.";
674 ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
675 "{modifies-per-second}", description,
676 1, Integer.MAX_VALUE);
677 ratePerSecond.setArgumentGroupName("Rate Management Arguments");
678 ratePerSecond.addLongIdentifier("rate-per-second");
679 parser.addArgument(ratePerSecond);
680
681 final String variableRateDataArgName = "variableRateData";
682 final String generateSampleRateFileArgName = "generateSampleRateFile";
683 description = RateAdjustor.getVariableRateDataArgumentDescription(
684 generateSampleRateFileArgName);
685 variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
686 "{path}", description, true, true, true,
687 false);
688 variableRateData.setArgumentGroupName("Rate Management Arguments");
689 variableRateData.addLongIdentifier("variable-rate-data");
690 parser.addArgument(variableRateData);
691
692 description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
693 variableRateDataArgName);
694 sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
695 false, 1, "{path}", description, false,
696 true, true, false);
697 sampleRateFile.setArgumentGroupName("Rate Management Arguments");
698 sampleRateFile.addLongIdentifier("generate-sample-rate-file");
699 sampleRateFile.setUsageArgument(true);
700 parser.addArgument(sampleRateFile);
701 parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
702
703 description = "The number of intervals to complete before beginning " +
704 "overall statistics collection. Specifying a nonzero " +
705 "number of warm-up intervals gives the client and server " +
706 "a chance to warm up without skewing performance results.";
707 warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
708 "{num}", description, 0, Integer.MAX_VALUE, 0);
709 warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
710 warmUpIntervals.addLongIdentifier("warm-up-intervals");
711 parser.addArgument(warmUpIntervals);
712
713 description = "Indicates the format to use for timestamps included in " +
714 "the output. A value of 'none' indicates that no " +
715 "timestamps should be included. A value of 'with-date' " +
716 "indicates that both the date and the time should be " +
717 "included. A value of 'without-date' indicates that only " +
718 "the time should be included.";
719 final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
720 allowedFormats.add("none");
721 allowedFormats.add("with-date");
722 allowedFormats.add("without-date");
723 timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
724 "{format}", description, allowedFormats, "none");
725 timestampFormat.addLongIdentifier("timestamp-format");
726 parser.addArgument(timestampFormat);
727
728 description = "Indicates that information about the result codes for " +
729 "failed operations should not be displayed.";
730 suppressErrorsArgument = new BooleanArgument(null,
731 "suppressErrorResultCodes", 1, description);
732 suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes");
733 parser.addArgument(suppressErrorsArgument);
734
735 description = "Generate output in CSV format rather than a " +
736 "display-friendly format";
737 csvFormat = new BooleanArgument('c', "csv", 1, description);
738 parser.addArgument(csvFormat);
739
740 description = "Specifies the seed to use for the random number generator.";
741 randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
742 description);
743 randomSeed.addLongIdentifier("random-seed");
744 parser.addArgument(randomSeed);
745
746
747 // The incrementAmount argument can only be used if the increment argument
748 // is provided.
749 parser.addDependentArgumentSet(incrementAmount, increment);
750
751
752 // Neither the valueLength nor valueCount arguments can be used if the
753 // increment argument is provided.
754 parser.addExclusiveArgumentSet(increment, valueLength);
755 parser.addExclusiveArgumentSet(increment, valueCount);
756 }
757
758
759
760 /**
761 * Indicates whether this tool supports creating connections to multiple
762 * servers. If it is to support multiple servers, then the "--hostname" and
763 * "--port" arguments will be allowed to be provided multiple times, and
764 * will be required to be provided the same number of times. The same type of
765 * communication security and bind credentials will be used for all servers.
766 *
767 * @return {@code true} if this tool supports creating connections to
768 * multiple servers, or {@code false} if not.
769 */
770 @Override()
771 protected boolean supportsMultipleServers()
772 {
773 return true;
774 }
775
776
777
778 /**
779 * Retrieves the connection options that should be used for connections
780 * created for use with this tool.
781 *
782 * @return The connection options that should be used for connections created
783 * for use with this tool.
784 */
785 @Override()
786 public LDAPConnectionOptions getConnectionOptions()
787 {
788 final LDAPConnectionOptions options = new LDAPConnectionOptions();
789 options.setUseSynchronousMode(true);
790 return options;
791 }
792
793
794
795 /**
796 * Performs the actual processing for this tool. In this case, it gets a
797 * connection to the directory server and uses it to perform the requested
798 * modifications.
799 *
800 * @return The result code for the processing that was performed.
801 */
802 @Override()
803 public ResultCode doToolProcessing()
804 {
805 runningThread = Thread.currentThread();
806
807 try
808 {
809 return doToolProcessingInternal();
810 }
811 finally
812 {
813 runningThread = null;
814 }
815
816 }
817
818
819 /**
820 * Performs the actual processing for this tool. In this case, it gets a
821 * connection to the directory server and uses it to perform the requested
822 * modifications.
823 *
824 * @return The result code for the processing that was performed.
825 */
826 private ResultCode doToolProcessingInternal()
827 {
828 // If the sample rate file argument was specified, then generate the sample
829 // variable rate data file and return.
830 if (sampleRateFile.isPresent())
831 {
832 try
833 {
834 RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
835 return ResultCode.SUCCESS;
836 }
837 catch (final Exception e)
838 {
839 debugException(e);
840 err("An error occurred while trying to write sample variable data " +
841 "rate file '", sampleRateFile.getValue().getAbsolutePath(),
842 "': ", getExceptionMessage(e));
843 return ResultCode.LOCAL_ERROR;
844 }
845 }
846
847
848 // Determine the random seed to use.
849 final Long seed;
850 if (randomSeed.isPresent())
851 {
852 seed = Long.valueOf(randomSeed.getValue());
853 }
854 else
855 {
856 seed = null;
857 }
858
859 // Create the value patterns for the target entry DN and proxied
860 // authorization identities.
861 final ValuePattern dnPattern;
862 try
863 {
864 dnPattern = new ValuePattern(entryDN.getValue(), seed);
865 }
866 catch (final ParseException pe)
867 {
868 debugException(pe);
869 err("Unable to parse the entry DN value pattern: ", pe.getMessage());
870 return ResultCode.PARAM_ERROR;
871 }
872
873 final ValuePattern authzIDPattern;
874 if (proxyAs.isPresent())
875 {
876 try
877 {
878 authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
879 }
880 catch (final ParseException pe)
881 {
882 debugException(pe);
883 err("Unable to parse the proxied authorization pattern: ",
884 pe.getMessage());
885 return ResultCode.PARAM_ERROR;
886 }
887 }
888 else
889 {
890 authzIDPattern = null;
891 }
892
893
894 // Get the set of controls to include in modify requests.
895 final ArrayList<Control> controlList = new ArrayList<Control>(5);
896 if (assertionFilter.isPresent())
897 {
898 controlList.add(new AssertionRequestControl(assertionFilter.getValue()));
899 }
900
901 if (permissiveModify.isPresent())
902 {
903 controlList.add(new PermissiveModifyRequestControl());
904 }
905
906 if (preReadAttribute.isPresent())
907 {
908 final List<String> attrList = preReadAttribute.getValues();
909 final String[] attrArray = new String[attrList.size()];
910 attrList.toArray(attrArray);
911 controlList.add(new PreReadRequestControl(attrArray));
912 }
913
914 if (postReadAttribute.isPresent())
915 {
916 final List<String> attrList = postReadAttribute.getValues();
917 final String[] attrArray = new String[attrList.size()];
918 attrList.toArray(attrArray);
919 controlList.add(new PostReadRequestControl(attrArray));
920 }
921
922 if (control.isPresent())
923 {
924 controlList.addAll(control.getValues());
925 }
926
927 final Control[] controlArray = new Control[controlList.size()];
928 controlList.toArray(controlArray);
929
930
931 // Get the names of the attributes to modify.
932 final String[] attrs = new String[attribute.getValues().size()];
933 attribute.getValues().toArray(attrs);
934
935
936 // Get the character set as a byte array.
937 final byte[] charSet = getBytes(characterSet.getValue());
938
939
940 // If the --ratePerSecond option was specified, then limit the rate
941 // accordingly.
942 FixedRateBarrier fixedRateBarrier = null;
943 if (ratePerSecond.isPresent() || variableRateData.isPresent())
944 {
945 // We might not have a rate per second if --variableRateData is specified.
946 // The rate typically doesn't matter except when we have warm-up
947 // intervals. In this case, we'll run at the max rate.
948 final int intervalSeconds = collectionInterval.getValue();
949 final int ratePerInterval =
950 (ratePerSecond.getValue() == null)
951 ? Integer.MAX_VALUE
952 : ratePerSecond.getValue() * intervalSeconds;
953 fixedRateBarrier =
954 new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
955 }
956
957
958 // If --variableRateData was specified, then initialize a RateAdjustor.
959 RateAdjustor rateAdjustor = null;
960 if (variableRateData.isPresent())
961 {
962 try
963 {
964 rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
965 ratePerSecond.getValue(), variableRateData.getValue());
966 }
967 catch (final IOException e)
968 {
969 debugException(e);
970 err("Initializing the variable rates failed: " + e.getMessage());
971 return ResultCode.PARAM_ERROR;
972 }
973 catch (final IllegalArgumentException e)
974 {
975 debugException(e);
976 err("Initializing the variable rates failed: " + e.getMessage());
977 return ResultCode.PARAM_ERROR;
978 }
979 }
980
981
982 // Determine whether to include timestamps in the output and if so what
983 // format should be used for them.
984 final boolean includeTimestamp;
985 final String timeFormat;
986 if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
987 {
988 includeTimestamp = true;
989 timeFormat = "dd/MM/yyyy HH:mm:ss";
990 }
991 else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
992 {
993 includeTimestamp = true;
994 timeFormat = "HH:mm:ss";
995 }
996 else
997 {
998 includeTimestamp = false;
999 timeFormat = null;
1000 }
1001
1002
1003 // Determine whether any warm-up intervals should be run.
1004 final long totalIntervals;
1005 final boolean warmUp;
1006 int remainingWarmUpIntervals = warmUpIntervals.getValue();
1007 if (remainingWarmUpIntervals > 0)
1008 {
1009 warmUp = true;
1010 totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
1011 }
1012 else
1013 {
1014 warmUp = true;
1015 totalIntervals = 0L + numIntervals.getValue();
1016 }
1017
1018
1019 // Create the table that will be used to format the output.
1020 final OutputFormat outputFormat;
1021 if (csvFormat.isPresent())
1022 {
1023 outputFormat = OutputFormat.CSV;
1024 }
1025 else
1026 {
1027 outputFormat = OutputFormat.COLUMNS;
1028 }
1029
1030 final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
1031 timeFormat, outputFormat, " ",
1032 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1033 "Mods/Sec"),
1034 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1035 "Avg Dur ms"),
1036 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1037 "Errors/Sec"),
1038 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1039 "Mods/Sec"),
1040 new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1041 "Avg Dur ms"));
1042
1043
1044 // Create values to use for statistics collection.
1045 final AtomicLong modCounter = new AtomicLong(0L);
1046 final AtomicLong errorCounter = new AtomicLong(0L);
1047 final AtomicLong modDurations = new AtomicLong(0L);
1048 final ResultCodeCounter rcCounter = new ResultCodeCounter();
1049
1050
1051 // Determine the length of each interval in milliseconds.
1052 final long intervalMillis = 1000L * collectionInterval.getValue();
1053
1054
1055 // Create a random number generator to use for seeding the per-thread
1056 // generators.
1057 final Random random = new Random();
1058
1059
1060 // Create the threads to use for the modifications.
1061 final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
1062 final ModRateThread[] threads = new ModRateThread[numThreads.getValue()];
1063 for (int i=0; i < threads.length; i++)
1064 {
1065 final LDAPConnection connection;
1066 try
1067 {
1068 connection = getConnection();
1069 }
1070 catch (final LDAPException le)
1071 {
1072 debugException(le);
1073 err("Unable to connect to the directory server: ",
1074 getExceptionMessage(le));
1075 return le.getResultCode();
1076 }
1077
1078 threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs,
1079 charSet, valueLength.getValue(), valueCount.getValue(),
1080 increment.isPresent(), incrementAmount.getValue(), controlArray,
1081 authzIDPattern, random.nextLong(),
1082 iterationsBeforeReconnect.getValue(), barrier, modCounter,
1083 modDurations, errorCounter, rcCounter, fixedRateBarrier);
1084 threads[i].start();
1085 }
1086
1087
1088 // Display the table header.
1089 for (final String headerLine : formatter.getHeaderLines(true))
1090 {
1091 out(headerLine);
1092 }
1093
1094
1095 // Start the RateAdjustor before the threads so that the initial value is
1096 // in place before any load is generated unless we're doing a warm-up in
1097 // which case, we'll start it after the warm-up is complete.
1098 if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1099 {
1100 rateAdjustor.start();
1101 }
1102
1103
1104 // Indicate that the threads can start running.
1105 try
1106 {
1107 barrier.await();
1108 }
1109 catch (final Exception e)
1110 {
1111 debugException(e);
1112 }
1113
1114 long overallStartTime = System.nanoTime();
1115 long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1116
1117
1118 boolean setOverallStartTime = false;
1119 long lastDuration = 0L;
1120 long lastNumErrors = 0L;
1121 long lastNumMods = 0L;
1122 long lastEndTime = System.nanoTime();
1123 for (long i=0; i < totalIntervals; i++)
1124 {
1125 if (rateAdjustor != null)
1126 {
1127 if (! rateAdjustor.isAlive())
1128 {
1129 out("All of the rates in " + variableRateData.getValue().getName() +
1130 " have been completed.");
1131 break;
1132 }
1133 }
1134
1135 final long startTimeMillis = System.currentTimeMillis();
1136 final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1137 nextIntervalStartTime += intervalMillis;
1138 if (sleepTimeMillis > 0)
1139 {
1140 sleeper.sleep(sleepTimeMillis);
1141 }
1142
1143 if (stopRequested.get())
1144 {
1145 break;
1146 }
1147
1148 final long endTime = System.nanoTime();
1149 final long intervalDuration = endTime - lastEndTime;
1150
1151 final long numMods;
1152 final long numErrors;
1153 final long totalDuration;
1154 if (warmUp && (remainingWarmUpIntervals > 0))
1155 {
1156 numMods = modCounter.getAndSet(0L);
1157 numErrors = errorCounter.getAndSet(0L);
1158 totalDuration = modDurations.getAndSet(0L);
1159 }
1160 else
1161 {
1162 numMods = modCounter.get();
1163 numErrors = errorCounter.get();
1164 totalDuration = modDurations.get();
1165 }
1166
1167 final long recentNumMods = numMods - lastNumMods;
1168 final long recentNumErrors = numErrors - lastNumErrors;
1169 final long recentDuration = totalDuration - lastDuration;
1170
1171 final double numSeconds = intervalDuration / 1000000000.0d;
1172 final double recentModRate = recentNumMods / numSeconds;
1173 final double recentErrorRate = recentNumErrors / numSeconds;
1174
1175 final double recentAvgDuration;
1176 if (recentNumMods > 0L)
1177 {
1178 recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1000000;
1179 }
1180 else
1181 {
1182 recentAvgDuration = 0.0d;
1183 }
1184
1185 if (warmUp && (remainingWarmUpIntervals > 0))
1186 {
1187 out(formatter.formatRow(recentModRate, recentAvgDuration,
1188 recentErrorRate, "warming up", "warming up"));
1189
1190 remainingWarmUpIntervals--;
1191 if (remainingWarmUpIntervals == 0)
1192 {
1193 out("Warm-up completed. Beginning overall statistics collection.");
1194 setOverallStartTime = true;
1195 if (rateAdjustor != null)
1196 {
1197 rateAdjustor.start();
1198 }
1199 }
1200 }
1201 else
1202 {
1203 if (setOverallStartTime)
1204 {
1205 overallStartTime = lastEndTime;
1206 setOverallStartTime = false;
1207 }
1208
1209 final double numOverallSeconds =
1210 (endTime - overallStartTime) / 1000000000.0d;
1211 final double overallAuthRate = numMods / numOverallSeconds;
1212
1213 final double overallAvgDuration;
1214 if (numMods > 0L)
1215 {
1216 overallAvgDuration = 1.0d * totalDuration / numMods / 1000000;
1217 }
1218 else
1219 {
1220 overallAvgDuration = 0.0d;
1221 }
1222
1223 out(formatter.formatRow(recentModRate, recentAvgDuration,
1224 recentErrorRate, overallAuthRate, overallAvgDuration));
1225
1226 lastNumMods = numMods;
1227 lastNumErrors = numErrors;
1228 lastDuration = totalDuration;
1229 }
1230
1231 final List<ObjectPair<ResultCode,Long>> rcCounts =
1232 rcCounter.getCounts(true);
1233 if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty()))
1234 {
1235 err("\tError Results:");
1236 for (final ObjectPair<ResultCode,Long> p : rcCounts)
1237 {
1238 err("\t", p.getFirst().getName(), ": ", p.getSecond());
1239 }
1240 }
1241
1242 lastEndTime = endTime;
1243 }
1244
1245 // Shut down the RateAdjustor if we have one.
1246 if (rateAdjustor != null)
1247 {
1248 rateAdjustor.shutDown();
1249 }
1250
1251 // Stop all of the threads.
1252 ResultCode resultCode = ResultCode.SUCCESS;
1253 for (final ModRateThread t : threads)
1254 {
1255 final ResultCode r = t.stopRunning();
1256 if (resultCode == ResultCode.SUCCESS)
1257 {
1258 resultCode = r;
1259 }
1260 }
1261
1262 return resultCode;
1263 }
1264
1265
1266
1267 /**
1268 * Requests that this tool stop running. This method will attempt to wait
1269 * for all threads to complete before returning control to the caller.
1270 */
1271 public void stopRunning()
1272 {
1273 stopRequested.set(true);
1274 sleeper.wakeup();
1275
1276 final Thread t = runningThread;
1277 if (t != null)
1278 {
1279 try
1280 {
1281 t.join();
1282 }
1283 catch (final Exception e)
1284 {
1285 debugException(e);
1286 }
1287 }
1288 }
1289
1290
1291
1292 /**
1293 * {@inheritDoc}
1294 */
1295 @Override()
1296 public LinkedHashMap<String[],String> getExampleUsages()
1297 {
1298 final LinkedHashMap<String[],String> examples =
1299 new LinkedHashMap<String[],String>(2);
1300
1301 String[] args =
1302 {
1303 "--hostname", "server.example.com",
1304 "--port", "389",
1305 "--bindDN", "uid=admin,dc=example,dc=com",
1306 "--bindPassword", "password",
1307 "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com",
1308 "--attribute", "description",
1309 "--valueLength", "12",
1310 "--numThreads", "10"
1311 };
1312 String description =
1313 "Test modify performance by randomly selecting entries across a set " +
1314 "of one million users located below 'ou=People,dc=example,dc=com' " +
1315 "with ten concurrent threads and replacing the values for the " +
1316 "description attribute with a string of 12 randomly-selected " +
1317 "lowercase alphabetic characters.";
1318 examples.put(args, description);
1319
1320 args = new String[]
1321 {
1322 "--generateSampleRateFile", "variable-rate-data.txt"
1323 };
1324 description =
1325 "Generate a sample variable rate definition file that may be used " +
1326 "in conjunction with the --variableRateData argument. The sample " +
1327 "file will include comments that describe the format for data to be " +
1328 "included in this file.";
1329 examples.put(args, description);
1330
1331 return examples;
1332 }
1333 }