001/*
002 * Copyright 2013-2020 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2013-2020 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2013-2020 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.examples;
037
038
039
040import java.io.OutputStream;
041import java.util.Collections;
042import java.util.LinkedHashMap;
043import java.util.List;
044import java.util.Map;
045import java.util.TreeMap;
046import java.util.concurrent.atomic.AtomicLong;
047
048import com.unboundid.asn1.ASN1OctetString;
049import com.unboundid.ldap.sdk.Attribute;
050import com.unboundid.ldap.sdk.DN;
051import com.unboundid.ldap.sdk.Filter;
052import com.unboundid.ldap.sdk.LDAPConnectionOptions;
053import com.unboundid.ldap.sdk.LDAPConnectionPool;
054import com.unboundid.ldap.sdk.LDAPException;
055import com.unboundid.ldap.sdk.LDAPSearchException;
056import com.unboundid.ldap.sdk.ResultCode;
057import com.unboundid.ldap.sdk.SearchRequest;
058import com.unboundid.ldap.sdk.SearchResult;
059import com.unboundid.ldap.sdk.SearchResultEntry;
060import com.unboundid.ldap.sdk.SearchResultReference;
061import com.unboundid.ldap.sdk.SearchResultListener;
062import com.unboundid.ldap.sdk.SearchScope;
063import com.unboundid.ldap.sdk.Version;
064import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
065import com.unboundid.util.Debug;
066import com.unboundid.util.LDAPCommandLineTool;
067import com.unboundid.util.StaticUtils;
068import com.unboundid.util.ThreadSafety;
069import com.unboundid.util.ThreadSafetyLevel;
070import com.unboundid.util.args.ArgumentException;
071import com.unboundid.util.args.ArgumentParser;
072import com.unboundid.util.args.DNArgument;
073import com.unboundid.util.args.IntegerArgument;
074import com.unboundid.util.args.StringArgument;
075
076
077
078/**
079 * This class provides a tool that may be used to identify references to entries
080 * that do not exist.  This tool can be useful for verifying existing data in
081 * directory servers that provide support for referential integrity.
082 * <BR><BR>
083 * All of the necessary information is provided using command line arguments.
084 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
085 * class, as well as the following additional arguments:
086 * <UL>
087 *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
088 *       for the searches.  At least one base DN must be provided.</LI>
089 *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
090 *       that is expected to contain references to other entries.  This
091 *       attribute should be indexed for equality searches, and its values
092 *       should be DNs.  At least one attribute must be provided.</LI>
093 *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
094 *       to find entries with references to other entries should use the simple
095 *       paged results control to iterate across entries in fixed-size pages
096 *       rather than trying to use a single search to identify all entries that
097 *       reference other entries.</LI>
098 * </UL>
099 */
100@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
101public final class IdentifyReferencesToMissingEntries
102       extends LDAPCommandLineTool
103       implements SearchResultListener
104{
105  /**
106   * The serial version UID for this serializable class.
107   */
108  private static final long serialVersionUID = 1981894839719501258L;
109
110
111
112  // The number of entries examined so far.
113  private final AtomicLong entriesExamined;
114
115  // The argument used to specify the base DNs to use for searches.
116  private DNArgument baseDNArgument;
117
118  // The argument used to specify the search page size.
119  private IntegerArgument pageSizeArgument;
120
121  // The connection to use for retrieving referenced entries.
122  private LDAPConnectionPool getReferencedEntriesPool;
123
124  // A map with counts of missing references by attribute type.
125  private final Map<String,AtomicLong> missingReferenceCounts;
126
127  // The names of the attributes for which to find missing references.
128  private String[] attributes;
129
130  // The argument used to specify the attributes for which to find missing
131  // references.
132  private StringArgument attributeArgument;
133
134
135
136  /**
137   * Parse the provided command line arguments and perform the appropriate
138   * processing.
139   *
140   * @param  args  The command line arguments provided to this program.
141   */
142  public static void main(final String... args)
143  {
144    final ResultCode resultCode = main(args, System.out, System.err);
145    if (resultCode != ResultCode.SUCCESS)
146    {
147      System.exit(resultCode.intValue());
148    }
149  }
150
151
152
153  /**
154   * Parse the provided command line arguments and perform the appropriate
155   * processing.
156   *
157   * @param  args       The command line arguments provided to this program.
158   * @param  outStream  The output stream to which standard out should be
159   *                    written.  It may be {@code null} if output should be
160   *                    suppressed.
161   * @param  errStream  The output stream to which standard error should be
162   *                    written.  It may be {@code null} if error messages
163   *                    should be suppressed.
164   *
165   * @return A result code indicating whether the processing was successful.
166   */
167  public static ResultCode main(final String[] args,
168                                final OutputStream outStream,
169                                final OutputStream errStream)
170  {
171    final IdentifyReferencesToMissingEntries tool =
172         new IdentifyReferencesToMissingEntries(outStream, errStream);
173    return tool.runTool(args);
174  }
175
176
177
178  /**
179   * Creates a new instance of this tool.
180   *
181   * @param  outStream  The output stream to which standard out should be
182   *                    written.  It may be {@code null} if output should be
183   *                    suppressed.
184   * @param  errStream  The output stream to which standard error should be
185   *                    written.  It may be {@code null} if error messages
186   *                    should be suppressed.
187   */
188  public IdentifyReferencesToMissingEntries(final OutputStream outStream,
189                                            final OutputStream errStream)
190  {
191    super(outStream, errStream);
192
193    baseDNArgument = null;
194    pageSizeArgument = null;
195    attributeArgument = null;
196    getReferencedEntriesPool = null;
197
198    entriesExamined = new AtomicLong(0L);
199    missingReferenceCounts = new TreeMap<>();
200  }
201
202
203
204  /**
205   * Retrieves the name of this tool.  It should be the name of the command used
206   * to invoke this tool.
207   *
208   * @return  The name for this tool.
209   */
210  @Override()
211  public String getToolName()
212  {
213    return "identify-references-to-missing-entries";
214  }
215
216
217
218  /**
219   * Retrieves a human-readable description for this tool.
220   *
221   * @return  A human-readable description for this tool.
222   */
223  @Override()
224  public String getToolDescription()
225  {
226    return "This tool may be used to identify entries containing one or more " +
227         "attributes which reference entries that do not exist.  This may " +
228         "require the ability to perform unindexed searches and/or the " +
229         "ability to use the simple paged results control.";
230  }
231
232
233
234  /**
235   * Retrieves a version string for this tool, if available.
236   *
237   * @return  A version string for this tool, or {@code null} if none is
238   *          available.
239   */
240  @Override()
241  public String getToolVersion()
242  {
243    return Version.NUMERIC_VERSION_STRING;
244  }
245
246
247
248  /**
249   * Indicates whether this tool should provide support for an interactive mode,
250   * in which the tool offers a mode in which the arguments can be provided in
251   * a text-driven menu rather than requiring them to be given on the command
252   * line.  If interactive mode is supported, it may be invoked using the
253   * "--interactive" argument.  Alternately, if interactive mode is supported
254   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
255   * interactive mode may be invoked by simply launching the tool without any
256   * arguments.
257   *
258   * @return  {@code true} if this tool supports interactive mode, or
259   *          {@code false} if not.
260   */
261  @Override()
262  public boolean supportsInteractiveMode()
263  {
264    return true;
265  }
266
267
268
269  /**
270   * Indicates whether this tool defaults to launching in interactive mode if
271   * the tool is invoked without any command-line arguments.  This will only be
272   * used if {@link #supportsInteractiveMode()} returns {@code true}.
273   *
274   * @return  {@code true} if this tool defaults to using interactive mode if
275   *          launched without any command-line arguments, or {@code false} if
276   *          not.
277   */
278  @Override()
279  public boolean defaultsToInteractiveMode()
280  {
281    return true;
282  }
283
284
285
286  /**
287   * Indicates whether this tool should provide arguments for redirecting output
288   * to a file.  If this method returns {@code true}, then the tool will offer
289   * an "--outputFile" argument that will specify the path to a file to which
290   * all standard output and standard error content will be written, and it will
291   * also offer a "--teeToStandardOut" argument that can only be used if the
292   * "--outputFile" argument is present and will cause all output to be written
293   * to both the specified output file and to standard output.
294   *
295   * @return  {@code true} if this tool should provide arguments for redirecting
296   *          output to a file, or {@code false} if not.
297   */
298  @Override()
299  protected boolean supportsOutputFile()
300  {
301    return true;
302  }
303
304
305
306  /**
307   * Indicates whether this tool should default to interactively prompting for
308   * the bind password if a password is required but no argument was provided
309   * to indicate how to get the password.
310   *
311   * @return  {@code true} if this tool should default to interactively
312   *          prompting for the bind password, or {@code false} if not.
313   */
314  @Override()
315  protected boolean defaultToPromptForBindPassword()
316  {
317    return true;
318  }
319
320
321
322  /**
323   * Indicates whether this tool supports the use of a properties file for
324   * specifying default values for arguments that aren't specified on the
325   * command line.
326   *
327   * @return  {@code true} if this tool supports the use of a properties file
328   *          for specifying default values for arguments that aren't specified
329   *          on the command line, or {@code false} if not.
330   */
331  @Override()
332  public boolean supportsPropertiesFile()
333  {
334    return true;
335  }
336
337
338
339  /**
340   * Indicates whether the LDAP-specific arguments should include alternate
341   * versions of all long identifiers that consist of multiple words so that
342   * they are available in both camelCase and dash-separated versions.
343   *
344   * @return  {@code true} if this tool should provide multiple versions of
345   *          long identifiers for LDAP-specific arguments, or {@code false} if
346   *          not.
347   */
348  @Override()
349  protected boolean includeAlternateLongIdentifiers()
350  {
351    return true;
352  }
353
354
355
356  /**
357   * Indicates whether this tool should provide a command-line argument that
358   * allows for low-level SSL debugging.  If this returns {@code true}, then an
359   * "--enableSSLDebugging}" argument will be added that sets the
360   * "javax.net.debug" system property to "all" before attempting any
361   * communication.
362   *
363   * @return  {@code true} if this tool should offer an "--enableSSLDebugging"
364   *          argument, or {@code false} if not.
365   */
366  @Override()
367  protected boolean supportsSSLDebugging()
368  {
369    return true;
370  }
371
372
373
374  /**
375   * Adds the arguments needed by this command-line tool to the provided
376   * argument parser which are not related to connecting or authenticating to
377   * the directory server.
378   *
379   * @param  parser  The argument parser to which the arguments should be added.
380   *
381   * @throws  ArgumentException  If a problem occurs while adding the arguments.
382   */
383  @Override()
384  public void addNonLDAPArguments(final ArgumentParser parser)
385         throws ArgumentException
386  {
387    String description = "The search base DN(s) to use to find entries with " +
388         "references to other entries.  At least one base DN must be " +
389         "specified.";
390    baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
391         description);
392    baseDNArgument.addLongIdentifier("base-dn", true);
393    parser.addArgument(baseDNArgument);
394
395    description = "The attribute(s) for which to find missing references.  " +
396         "At least one attribute must be specified, and each attribute " +
397         "must be indexed for equality searches and have values which are DNs.";
398    attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
399         description);
400    parser.addArgument(attributeArgument);
401
402    description = "The maximum number of entries to retrieve at a time when " +
403         "attempting to find entries with references to other entries.  This " +
404         "requires that the authenticated user have permission to use the " +
405         "simple paged results control, but it can avoid problems with the " +
406         "server sending entries too quickly for the client to handle.  By " +
407         "default, the simple paged results control will not be used.";
408    pageSizeArgument =
409         new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
410              description, 1, Integer.MAX_VALUE);
411    pageSizeArgument.addLongIdentifier("simple-page-size", true);
412    parser.addArgument(pageSizeArgument);
413  }
414
415
416
417  /**
418   * Retrieves the connection options that should be used for connections that
419   * are created with this command line tool.  Subclasses may override this
420   * method to use a custom set of connection options.
421   *
422   * @return  The connection options that should be used for connections that
423   *          are created with this command line tool.
424   */
425  @Override()
426  public LDAPConnectionOptions getConnectionOptions()
427  {
428    final LDAPConnectionOptions options = new LDAPConnectionOptions();
429
430    options.setUseSynchronousMode(true);
431    options.setResponseTimeoutMillis(0L);
432
433    return options;
434  }
435
436
437
438  /**
439   * Performs the core set of processing for this tool.
440   *
441   * @return  A result code that indicates whether the processing completed
442   *          successfully.
443   */
444  @Override()
445  public ResultCode doToolProcessing()
446  {
447    // Establish a connection to the target directory server to use for
448    // finding references to entries.
449    final LDAPConnectionPool findReferencesPool;
450    try
451    {
452      findReferencesPool = getConnectionPool(1, 1);
453      findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true);
454    }
455    catch (final LDAPException le)
456    {
457      Debug.debugException(le);
458      err("Unable to establish a connection to the directory server:  ",
459           StaticUtils.getExceptionMessage(le));
460      return le.getResultCode();
461    }
462
463    try
464    {
465      // Establish a second connection to use for retrieving referenced entries.
466      try
467      {
468        getReferencedEntriesPool = getConnectionPool(1,1);
469        getReferencedEntriesPool.
470             setRetryFailedOperationsDueToInvalidConnections(true);
471      }
472      catch (final LDAPException le)
473      {
474        Debug.debugException(le);
475        err("Unable to establish a connection to the directory server:  ",
476             StaticUtils.getExceptionMessage(le));
477        return le.getResultCode();
478      }
479
480
481      // Get the set of attributes for which to find missing references.
482      final List<String> attrList = attributeArgument.getValues();
483      attributes = new String[attrList.size()];
484      attrList.toArray(attributes);
485
486
487      // Construct a search filter that will be used to find all entries with
488      // references to other entries.
489      final Filter filter;
490      if (attributes.length == 1)
491      {
492        filter = Filter.createPresenceFilter(attributes[0]);
493        missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
494      }
495      else
496      {
497        final Filter[] orComps = new Filter[attributes.length];
498        for (int i=0; i < attributes.length; i++)
499        {
500          orComps[i] = Filter.createPresenceFilter(attributes[i]);
501          missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
502        }
503        filter = Filter.createORFilter(orComps);
504      }
505
506
507      // Iterate across all of the search base DNs and perform searches to find
508      // missing references.
509      for (final DN baseDN : baseDNArgument.getValues())
510      {
511        ASN1OctetString cookie = null;
512        do
513        {
514          final SearchRequest searchRequest = new SearchRequest(this,
515               baseDN.toString(), SearchScope.SUB, filter, attributes);
516          if (pageSizeArgument.isPresent())
517          {
518            searchRequest.addControl(new SimplePagedResultsControl(
519                 pageSizeArgument.getValue(), cookie, false));
520          }
521
522          SearchResult searchResult;
523          try
524          {
525            searchResult = findReferencesPool.search(searchRequest);
526          }
527          catch (final LDAPSearchException lse)
528          {
529            Debug.debugException(lse);
530            try
531            {
532              searchResult = findReferencesPool.search(searchRequest);
533            }
534            catch (final LDAPSearchException lse2)
535            {
536              Debug.debugException(lse2);
537              searchResult = lse2.getSearchResult();
538            }
539          }
540
541          if (searchResult.getResultCode() != ResultCode.SUCCESS)
542          {
543            err("An error occurred while attempting to search for missing " +
544                 "references to entries below " + baseDN + ":  " +
545                 searchResult.getDiagnosticMessage());
546            return searchResult.getResultCode();
547          }
548
549          final SimplePagedResultsControl pagedResultsResponse;
550          try
551          {
552            pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
553          }
554          catch (final LDAPException le)
555          {
556            Debug.debugException(le);
557            err("An error occurred while attempting to decode a simple " +
558                 "paged results response control in the response to a " +
559                 "search for entries below " + baseDN + ":  " +
560                 StaticUtils.getExceptionMessage(le));
561            return le.getResultCode();
562          }
563
564          if (pagedResultsResponse != null)
565          {
566            if (pagedResultsResponse.moreResultsToReturn())
567            {
568              cookie = pagedResultsResponse.getCookie();
569            }
570            else
571            {
572              cookie = null;
573            }
574          }
575        }
576        while (cookie != null);
577      }
578
579
580      // See if there were any missing references found.
581      boolean missingReferenceFound = false;
582      for (final Map.Entry<String,AtomicLong> e :
583           missingReferenceCounts.entrySet())
584      {
585        final long numMissing = e.getValue().get();
586        if (numMissing > 0L)
587        {
588          if (! missingReferenceFound)
589          {
590            err();
591            missingReferenceFound = true;
592          }
593
594          err("Found " + numMissing + ' ' + e.getKey() +
595               " references to entries that do not exist.");
596        }
597      }
598
599      if (missingReferenceFound)
600      {
601        return ResultCode.CONSTRAINT_VIOLATION;
602      }
603      else
604      {
605        out("No references were found to entries that do not exist.");
606        return ResultCode.SUCCESS;
607      }
608    }
609    finally
610    {
611      findReferencesPool.close();
612
613      if (getReferencedEntriesPool != null)
614      {
615        getReferencedEntriesPool.close();
616      }
617    }
618  }
619
620
621
622  /**
623   * Retrieves a map that correlates the number of missing references found by
624   * attribute type.
625   *
626   * @return  A map that correlates the number of missing references found by
627   *          attribute type.
628   */
629  public Map<String,AtomicLong> getMissingReferenceCounts()
630  {
631    return Collections.unmodifiableMap(missingReferenceCounts);
632  }
633
634
635
636  /**
637   * Retrieves a set of information that may be used to generate example usage
638   * information.  Each element in the returned map should consist of a map
639   * between an example set of arguments and a string that describes the
640   * behavior of the tool when invoked with that set of arguments.
641   *
642   * @return  A set of information that may be used to generate example usage
643   *          information.  It may be {@code null} or empty if no example usage
644   *          information is available.
645   */
646  @Override()
647  public LinkedHashMap<String[],String> getExampleUsages()
648  {
649    final LinkedHashMap<String[],String> exampleMap =
650         new LinkedHashMap<>(StaticUtils.computeMapCapacity(1));
651
652    final String[] args =
653    {
654      "--hostname", "server.example.com",
655      "--port", "389",
656      "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
657      "--bindPassword", "password",
658      "--baseDN", "dc=example,dc=com",
659      "--attribute", "member",
660      "--attribute", "uniqueMember",
661      "--simplePageSize", "100"
662    };
663    exampleMap.put(args,
664         "Identify all entries below dc=example,dc=com in which either the " +
665              "member or uniqueMember attribute references an entry that " +
666              "does not exist.");
667
668    return exampleMap;
669  }
670
671
672
673  /**
674   * Indicates that the provided search result entry has been returned by the
675   * server and may be processed by this search result listener.
676   *
677   * @param  searchEntry  The search result entry that has been returned by the
678   *                      server.
679   */
680  @Override()
681  public void searchEntryReturned(final SearchResultEntry searchEntry)
682  {
683    try
684    {
685      // Find attributes which references to entries that do not exist.
686      for (final String attr : attributes)
687      {
688        final List<Attribute> attrList =
689             searchEntry.getAttributesWithOptions(attr, null);
690        for (final Attribute a : attrList)
691        {
692          for (final String value : a.getValues())
693          {
694            try
695            {
696              final SearchResultEntry e =
697                   getReferencedEntriesPool.getEntry(value, "1.1");
698              if (e == null)
699              {
700                err("Entry '", searchEntry.getDN(), "' includes attribute ",
701                     a.getName(), " that references entry '", value,
702                     "' which does not exist.");
703                missingReferenceCounts.get(attr).incrementAndGet();
704              }
705            }
706            catch (final LDAPException le)
707            {
708              Debug.debugException(le);
709              err("An error occurred while attempting to determine whether " +
710                   "entry '" + value + "' referenced in attribute " +
711                   a.getName() + " of entry '" + searchEntry.getDN() +
712                   "' exists:  " + StaticUtils.getExceptionMessage(le));
713              missingReferenceCounts.get(attr).incrementAndGet();
714            }
715          }
716        }
717      }
718    }
719    finally
720    {
721      final long count = entriesExamined.incrementAndGet();
722      if ((count % 1000L) == 0L)
723      {
724        out(count, " entries examined");
725      }
726    }
727  }
728
729
730
731  /**
732   * Indicates that the provided search result reference has been returned by
733   * the server and may be processed by this search result listener.
734   *
735   * @param  searchReference  The search result reference that has been returned
736   *                          by the server.
737   */
738  @Override()
739  public void searchReferenceReturned(
740                   final SearchResultReference searchReference)
741  {
742    // No implementation is required.  This tool will not follow referrals.
743  }
744}