001 /*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License"). You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at
010 * trunk/opends/resource/legal-notices/OpenDS.LICENSE
011 * or https://OpenDS.dev.java.net/OpenDS.LICENSE.
012 * See the License for the specific language governing permissions
013 * and limitations under the License.
014 *
015 * When distributing Covered Code, include this CDDL HEADER in each
016 * file and include the License file at
017 * trunk/opends/resource/legal-notices/OpenDS.LICENSE. If applicable,
018 * add the following below this CDDL HEADER, with the fields enclosed
019 * by brackets "[]" replaced with your own identifying information:
020 * Portions Copyright [yyyy] [name of copyright owner]
021 *
022 * CDDL HEADER END
023 *
024 *
025 * Copyright 2006-2008 Sun Microsystems, Inc.
026 */
027 package org.opends.server.backends.jeb;
028 import org.opends.messages.Message;
029
030 import static org.opends.server.loggers.ErrorLogger.logError;
031 import static org.opends.server.loggers.debug.DebugLogger.*;
032 import org.opends.server.loggers.debug.DebugTracer;
033
034 import com.sleepycat.je.Cursor;
035 import com.sleepycat.je.CursorConfig;
036 import com.sleepycat.je.DatabaseEntry;
037 import com.sleepycat.je.DatabaseException;
038 import com.sleepycat.je.EnvironmentStats;
039 import com.sleepycat.je.LockMode;
040 import com.sleepycat.je.OperationStatus;
041 import com.sleepycat.je.StatsConfig;
042 import com.sleepycat.je.Transaction;
043
044 import org.opends.server.api.OrderingMatchingRule;
045 import org.opends.server.api.ApproximateMatchingRule;
046 import org.opends.server.core.DirectoryServer;
047 import org.opends.server.protocols.asn1.ASN1OctetString;
048 import org.opends.server.util.StaticUtils;
049 import org.opends.server.util.ServerConstants;
050
051 import org.opends.server.types.*;
052 import static org.opends.messages.JebMessages.*;
053 import java.util.ArrayList;
054 import java.util.Arrays;
055 import java.util.HashMap;
056 import java.util.IdentityHashMap;
057 import java.util.LinkedHashSet;
058 import java.util.List;
059 import java.util.Map;
060 import java.util.Set;
061 import java.util.Timer;
062 import java.util.TimerTask;
063
064 /**
065 * This class is used to run an index verification process on the backend.
066 */
067 public class VerifyJob
068 {
069 /**
070 * The tracer object for the debug logger.
071 */
072 private static final DebugTracer TRACER = getTracer();
073
074
075 /**
076 * The verify configuration.
077 */
078 private VerifyConfig verifyConfig;
079
080 /**
081 * The root container used for the verify job.
082 */
083 RootContainer rootContainer;
084
085 /**
086 * The number of milliseconds between job progress reports.
087 */
088 private long progressInterval = 10000;
089
090 /**
091 * The number of index keys processed.
092 */
093 private long keyCount = 0;
094
095 /**
096 * The number of errors found.
097 */
098 private long errorCount = 0;
099
100 /**
101 * The number of records that have exceeded the entry limit.
102 */
103 long entryLimitExceededCount = 0;
104
105 /**
106 * The number of records that reference more than one entry.
107 */
108 long multiReferenceCount = 0;
109
110 /**
111 * The total number of entry references.
112 */
113 long entryReferencesCount = 0;
114
115 /**
116 * The maximum number of references per record.
117 */
118 long maxEntryPerValue = 0;
119
120 /**
121 * This map is used to gather some statistics about values that have
122 * exceeded the entry limit.
123 */
124 IdentityHashMap<Index,HashMap<ByteString,Long>> entryLimitMap =
125 new IdentityHashMap<Index, HashMap<ByteString, Long>>();
126
127 /**
128 * Indicates whether the DN database is to be verified.
129 */
130 private boolean verifyDN2ID = false;
131
132 /**
133 * Indicates whether the children database is to be verified.
134 */
135 private boolean verifyID2Children = false;
136
137 /**
138 * Indicates whether the subtree database is to be verified.
139 */
140 private boolean verifyID2Subtree = false;
141
142 /**
143 * The entry database.
144 */
145 ID2Entry id2entry = null;
146
147 /**
148 * The DN database.
149 */
150 DN2ID dn2id = null;
151
152 /**
153 * The children database.
154 */
155 Index id2c = null;
156
157 /**
158 * The subtree database.
159 */
160 Index id2s = null;
161
162 /**
163 * A list of the attribute indexes to be verified.
164 */
165 ArrayList<AttributeIndex> attrIndexList = new ArrayList<AttributeIndex>();
166
167 /**
168 * A list of the VLV indexes to be verified.
169 */
170 ArrayList<VLVIndex> vlvIndexList = new ArrayList<VLVIndex>();
171
172 /**
173 * The types of indexes that are verifiable.
174 */
175 enum IndexType
176 {
177 PRES, EQ, SUBSTRING, ORDERING, APPROXIMATE
178 }
179
180 /**
181 * Construct a VerifyJob.
182 *
183 * @param verifyConfig The verify configuration.
184 */
185 public VerifyJob(VerifyConfig verifyConfig)
186 {
187 this.verifyConfig = verifyConfig;
188 }
189
190 /**
191 * Verify the backend.
192 *
193 * @param rootContainer The root container that holds the entries to verify.
194 * @param statEntry Optional statistics entry.
195 * @return The error count.
196 * @throws DatabaseException If an error occurs in the JE database.
197 * @throws JebException If an error occurs in the JE backend.
198 * @throws DirectoryException If an error occurs while verifying the backend.
199 */
200 public long verifyBackend(RootContainer rootContainer, Entry statEntry) throws
201 DatabaseException, JebException, DirectoryException
202 {
203 this.rootContainer = rootContainer;
204 EntryContainer entryContainer =
205 rootContainer.getEntryContainer(verifyConfig.getBaseDN());
206
207 entryContainer.sharedLock.lock();
208 try
209 {
210 ArrayList<String> completeList = verifyConfig.getCompleteList();
211 ArrayList<String> cleanList = verifyConfig.getCleanList();
212
213 boolean cleanMode = false;
214 if (completeList.isEmpty() && cleanList.isEmpty())
215 {
216 verifyDN2ID = true;
217 verifyID2Children = true;
218 verifyID2Subtree = true;
219 attrIndexList.addAll(entryContainer.getAttributeIndexes());
220 }
221 else
222 {
223 ArrayList<String> list;
224 if (!completeList.isEmpty())
225 {
226 list = completeList;
227 }
228 else
229 {
230 list = cleanList;
231 cleanMode = true;
232 }
233
234 for (String index : list)
235 {
236 String lowerName = index.toLowerCase();
237 if (lowerName.equals("dn2id"))
238 {
239 verifyDN2ID = true;
240 }
241 else if (lowerName.equals("id2children"))
242 {
243 verifyID2Children = true;
244 }
245 else if (lowerName.equals("id2subtree"))
246 {
247 verifyID2Subtree = true;
248 }
249 else if(lowerName.startsWith("vlv."))
250 {
251 if(lowerName.length() < 5)
252 {
253 Message msg = ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName);
254 throw new JebException(msg);
255 }
256
257 VLVIndex vlvIndex =
258 entryContainer.getVLVIndex(lowerName.substring(4));
259 if(vlvIndex == null)
260 {
261 Message msg =
262 ERR_JEB_VLV_INDEX_NOT_CONFIGURED.get(lowerName.substring(4));
263 throw new JebException(msg);
264 }
265
266 vlvIndexList.add(vlvIndex);
267 }
268 else
269 {
270 AttributeType attrType =
271 DirectoryServer.getAttributeType(lowerName);
272 if (attrType == null)
273 {
274 Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
275 throw new JebException(msg);
276 }
277 AttributeIndex attrIndex =
278 entryContainer.getAttributeIndex(attrType);
279 if (attrIndex == null)
280 {
281 Message msg = ERR_JEB_ATTRIBUTE_INDEX_NOT_CONFIGURED.get(index);
282 throw new JebException(msg);
283 }
284 attrIndexList.add(attrIndex);
285 }
286 }
287 }
288
289 entryLimitMap =
290 new IdentityHashMap<Index,HashMap<ByteString,Long>>(
291 attrIndexList.size());
292
293 // We will be updating these files independently of the indexes
294 // so we need direct access to them rather than going through
295 // the entry entryContainer methods.
296 id2entry = entryContainer.getID2Entry();
297 dn2id = entryContainer.getDN2ID();
298 id2c = entryContainer.getID2Children();
299 id2s = entryContainer.getID2Subtree();
300
301 // Make a note of the time we started.
302 long startTime = System.currentTimeMillis();
303
304 // Start a timer for the progress report.
305 Timer timer = new Timer();
306 TimerTask progressTask = new ProgressTask();
307 timer.scheduleAtFixedRate(progressTask, progressInterval,
308 progressInterval);
309
310 // Iterate through the index keys.
311 try
312 {
313 if (cleanMode)
314 {
315 iterateIndex();
316 }
317 else
318 {
319 iterateID2Entry();
320
321 // Make sure the vlv indexes are in correct order.
322 for(VLVIndex vlvIndex : vlvIndexList)
323 {
324 iterateVLVIndex(vlvIndex, false);
325 }
326 }
327 }
328 finally
329 {
330 timer.cancel();
331 }
332
333 long finishTime = System.currentTimeMillis();
334 long totalTime = (finishTime - startTime);
335
336 float rate = 0;
337 if (totalTime > 0)
338 {
339 rate = 1000f*keyCount / totalTime;
340 }
341
342 addStatEntry(statEntry, "verify-error-count",
343 String.valueOf(errorCount));
344 addStatEntry(statEntry, "verify-key-count",
345 String.valueOf(keyCount));
346 if (cleanMode)
347 {
348 Message message = NOTE_JEB_VERIFY_CLEAN_FINAL_STATUS.get(
349 keyCount, errorCount, totalTime/1000, rate);
350 logError(message);
351
352 if (multiReferenceCount > 0)
353 {
354 float averageEntryReferences = 0;
355 if (keyCount > 0)
356 {
357 averageEntryReferences = (float)entryReferencesCount/keyCount;
358 }
359
360 message =
361 INFO_JEB_VERIFY_MULTIPLE_REFERENCE_COUNT.get(multiReferenceCount);
362 logError(message);
363 addStatEntry(statEntry, "verify-multiple-reference-count",
364 String.valueOf(multiReferenceCount));
365
366 message = INFO_JEB_VERIFY_ENTRY_LIMIT_EXCEEDED_COUNT.get(
367 entryLimitExceededCount);
368 logError(message);
369 addStatEntry(statEntry, "verify-entry-limit-exceeded-count",
370 String.valueOf(entryLimitExceededCount));
371
372 message = INFO_JEB_VERIFY_AVERAGE_REFERENCE_COUNT.get(
373 averageEntryReferences);
374 logError(message);
375 addStatEntry(statEntry, "verify-average-reference-count",
376 String.valueOf(averageEntryReferences));
377
378 message =
379 INFO_JEB_VERIFY_MAX_REFERENCE_COUNT.get(maxEntryPerValue);
380 logError(message);
381 addStatEntry(statEntry, "verify-max-reference-count",
382 String.valueOf(maxEntryPerValue));
383 }
384 }
385 else
386 {
387 Message message = NOTE_JEB_VERIFY_FINAL_STATUS.get(
388 keyCount, errorCount, totalTime/1000, rate);
389 logError(message);
390 //TODO add entry-limit-stats to the statEntry
391 if (entryLimitMap.size() > 0)
392 {
393 message = INFO_JEB_VERIFY_ENTRY_LIMIT_STATS_HEADER.get();
394 logError(message);
395
396 for (Map.Entry<Index,HashMap<ByteString,Long>> mapEntry :
397 entryLimitMap.entrySet())
398 {
399 Index index = mapEntry.getKey();
400 Long[] values = mapEntry.getValue().values().toArray(new Long[0]);
401
402 // Calculate the median value for entry limit exceeded.
403 Arrays.sort(values);
404 long medianValue;
405 int x = values.length / 2;
406 if (values.length % 2 == 0)
407 {
408 medianValue = (values[x] + values[x-1]) / 2;
409 }
410 else
411 {
412 medianValue = values[x];
413 }
414
415 message = INFO_JEB_VERIFY_ENTRY_LIMIT_STATS_ROW.
416 get(index.toString(), values.length, values[0],
417 values[values.length-1], medianValue);
418 logError(message);
419 }
420 }
421 }
422 }
423 finally
424 {
425 entryContainer.sharedLock.unlock();
426 }
427 return errorCount;
428 }
429
430 /**
431 * Iterate through the entries in id2entry to perform a check for
432 * index completeness. We check that the ID for the entry is indeed
433 * present in the indexes for the appropriate values.
434 *
435 * @throws DatabaseException If an error occurs in the JE database.
436 */
437 private void iterateID2Entry() throws DatabaseException
438 {
439 Cursor cursor = id2entry.openCursor(null, new CursorConfig());
440 try
441 {
442 DatabaseEntry key = new DatabaseEntry();
443 DatabaseEntry data = new DatabaseEntry();
444
445 Long storedEntryCount = id2entry.getRecordCount();
446
447 OperationStatus status;
448 for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
449 status == OperationStatus.SUCCESS;
450 status = cursor.getNext(key, data, LockMode.DEFAULT))
451 {
452 EntryID entryID;
453 try
454 {
455 entryID = new EntryID(key);
456 }
457 catch (Exception e)
458 {
459 errorCount++;
460 if (debugEnabled())
461 {
462 TRACER.debugCaught(DebugLogLevel.ERROR, e);
463
464 TRACER.debugError("Malformed id2entry ID %s.%n",
465 StaticUtils.bytesToHex(key.getData()));
466 }
467 continue;
468 }
469
470 keyCount++;
471
472 Entry entry;
473 try
474 {
475 entry = JebFormat.entryFromDatabase(data.getData(),
476 rootContainer.getCompressedSchema());
477 }
478 catch (Exception e)
479 {
480 errorCount++;
481 if (debugEnabled())
482 {
483 TRACER.debugCaught(DebugLogLevel.ERROR, e);
484
485 TRACER.debugError("Malformed id2entry record for ID %d:%n%s%n",
486 entryID.longValue(),
487 StaticUtils.bytesToHex(data.getData()));
488 }
489 continue;
490 }
491
492 verifyEntry(entryID, entry);
493 }
494 if (keyCount != storedEntryCount)
495 {
496 errorCount++;
497 if (debugEnabled())
498 {
499 TRACER.debugError("The stored entry count in id2entry (%d) does " +
500 "not agree with the actual number of entry " +
501 "records found (%d).%n", storedEntryCount, keyCount);
502 }
503 }
504 }
505 finally
506 {
507 cursor.close();
508 }
509 }
510
511 /**
512 * Iterate through the entries in an index to perform a check for
513 * index cleanliness. For each ID in the index we check that the
514 * entry it refers to does indeed contain the expected value.
515 *
516 * @throws JebException If an error occurs in the JE backend.
517 * @throws DatabaseException If an error occurs in the JE database.
518 * @throws DirectoryException If an error occurs reading values in the index.
519 */
520 private void iterateIndex()
521 throws JebException, DatabaseException, DirectoryException
522 {
523 if (verifyDN2ID)
524 {
525 iterateDN2ID();
526 }
527 else if (verifyID2Children)
528 {
529 iterateID2Children();
530 }
531 else if (verifyID2Subtree)
532 {
533 iterateID2Subtree();
534 }
535 else
536 {
537 if(attrIndexList.size() > 0)
538 {
539 AttributeIndex attrIndex = attrIndexList.get(0);
540 iterateAttrIndex(attrIndex.getAttributeType(),
541 attrIndex.equalityIndex, IndexType.EQ );
542 iterateAttrIndex(attrIndex.getAttributeType(),
543 attrIndex.presenceIndex, IndexType.PRES);
544 iterateAttrIndex(attrIndex.getAttributeType(),
545 attrIndex.substringIndex, IndexType.SUBSTRING);
546 iterateAttrIndex(attrIndex.getAttributeType(),
547 attrIndex.orderingIndex, IndexType.ORDERING);
548 iterateAttrIndex(attrIndex.getAttributeType(),
549 attrIndex.approximateIndex, IndexType.APPROXIMATE);
550 } else if(vlvIndexList.size() > 0)
551 {
552 iterateVLVIndex(vlvIndexList.get(0), true);
553 }
554 }
555 }
556
557 /**
558 * Iterate through the entries in DN2ID to perform a check for
559 * index cleanliness.
560 *
561 * @throws DatabaseException If an error occurs in the JE database.
562 */
563 private void iterateDN2ID() throws DatabaseException
564 {
565 Cursor cursor = dn2id.openCursor(null, new CursorConfig());
566 try
567 {
568 DatabaseEntry key = new DatabaseEntry();
569 DatabaseEntry data = new DatabaseEntry();
570
571 OperationStatus status;
572 for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
573 status == OperationStatus.SUCCESS;
574 status = cursor.getNext(key, data, LockMode.DEFAULT))
575 {
576 keyCount++;
577
578 DN dn;
579 try
580 {
581 dn = DN.decode(new ASN1OctetString(key.getData()));
582 }
583 catch (DirectoryException e)
584 {
585 errorCount++;
586 if (debugEnabled())
587 {
588 TRACER.debugCaught(DebugLogLevel.ERROR, e);
589
590 TRACER.debugError("File dn2id has malformed key %s.%n",
591 StaticUtils.bytesToHex(key.getData()));
592 }
593 continue;
594 }
595
596 EntryID entryID;
597 try
598 {
599 entryID = new EntryID(data);
600 }
601 catch (Exception e)
602 {
603 errorCount++;
604 if (debugEnabled())
605 {
606 TRACER.debugCaught(DebugLogLevel.ERROR, e);
607
608 TRACER.debugError("File dn2id has malformed ID for DN <%s>:%n%s%n",
609 dn.toNormalizedString(),
610 StaticUtils.bytesToHex(data.getData()));
611 }
612 continue;
613 }
614
615 Entry entry;
616 try
617 {
618 entry = id2entry.get(null, entryID, LockMode.DEFAULT);
619 }
620 catch (Exception e)
621 {
622 errorCount++;
623 if (debugEnabled())
624 {
625 TRACER.debugCaught(DebugLogLevel.ERROR, e);
626 }
627 continue;
628 }
629
630 if (entry == null)
631 {
632 errorCount++;
633 if (debugEnabled())
634 {
635 TRACER.debugError("File dn2id has DN <%s> referencing unknown " +
636 "ID %d%n", dn.toNormalizedString(), entryID.longValue());
637 }
638 }
639 else
640 {
641 if (!entry.getDN().equals(dn))
642 {
643 errorCount++;
644 if (debugEnabled())
645 {
646 TRACER.debugError("File dn2id has DN <%s> referencing entry " +
647 "with wrong DN <%s>%n", dn.toNormalizedString(),
648 entry.getDN().toNormalizedString());
649 }
650 }
651 }
652 }
653 }
654 finally
655 {
656 cursor.close();
657 }
658 }
659
660 /**
661 * Iterate through the entries in ID2Children to perform a check for
662 * index cleanliness.
663 *
664 * @throws JebException If an error occurs in the JE backend.
665 * @throws DatabaseException If an error occurs in the JE database.
666 */
667 private void iterateID2Children() throws JebException, DatabaseException
668 {
669 Cursor cursor = id2c.openCursor(null, new CursorConfig());
670 try
671 {
672 DatabaseEntry key = new DatabaseEntry();
673 DatabaseEntry data = new DatabaseEntry();
674
675 OperationStatus status;
676 for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
677 status == OperationStatus.SUCCESS;
678 status = cursor.getNext(key, data, LockMode.DEFAULT))
679 {
680 keyCount++;
681
682 EntryID entryID;
683 try
684 {
685 entryID = new EntryID(key);
686 }
687 catch (Exception e)
688 {
689 errorCount++;
690 if (debugEnabled())
691 {
692 TRACER.debugCaught(DebugLogLevel.ERROR, e);
693
694 TRACER.debugError("File id2children has malformed ID %s%n",
695 StaticUtils.bytesToHex(key.getData()));
696 }
697 continue;
698 }
699
700 EntryIDSet entryIDList;
701
702 try
703 {
704 JebFormat.entryIDListFromDatabase(data.getData());
705 entryIDList = new EntryIDSet(key.getData(), data.getData());
706 }
707 catch (Exception e)
708 {
709 errorCount++;
710 if (debugEnabled())
711 {
712 TRACER.debugCaught(DebugLogLevel.ERROR, e);
713
714 TRACER.debugError("File id2children has malformed ID list " +
715 "for ID %s:%n%s%n", entryID,
716 StaticUtils.bytesToHex(data.getData()));
717 }
718 continue;
719 }
720
721 updateIndexStats(entryIDList);
722
723 if (entryIDList.isDefined())
724 {
725 Entry entry;
726 try
727 {
728 entry = id2entry.get(null, entryID, LockMode.DEFAULT);
729 }
730 catch (Exception e)
731 {
732 if (debugEnabled())
733 {
734 TRACER.debugCaught(DebugLogLevel.ERROR, e);
735 }
736 errorCount++;
737 continue;
738 }
739
740 if (entry == null)
741 {
742 errorCount++;
743 if (debugEnabled())
744 {
745 TRACER.debugError("File id2children has unknown ID %d%n",
746 entryID.longValue());
747 }
748 continue;
749 }
750
751 for (EntryID id : entryIDList)
752 {
753 Entry childEntry;
754 try
755 {
756 childEntry = id2entry.get(null, id, LockMode.DEFAULT);
757 }
758 catch (Exception e)
759 {
760 if (debugEnabled())
761 {
762 TRACER.debugCaught(DebugLogLevel.ERROR, e);
763 }
764 errorCount++;
765 continue;
766 }
767
768 if (childEntry == null)
769 {
770 errorCount++;
771 if (debugEnabled())
772 {
773 TRACER.debugError("File id2children has ID %d referencing " +
774 "unknown ID %d%n", entryID.longValue(), id.longValue());
775 }
776 continue;
777 }
778
779 if (!childEntry.getDN().isDescendantOf(entry.getDN()) ||
780 childEntry.getDN().getNumComponents() !=
781 entry.getDN().getNumComponents() + 1)
782 {
783 errorCount++;
784 if (debugEnabled())
785 {
786 TRACER.debugError("File id2children has ID %d with DN <%s> " +
787 "referencing ID %d with non-child DN <%s>%n",
788 entryID.longValue(), entry.getDN().toString(),
789 id.longValue(), childEntry.getDN().toString());
790 }
791 }
792 }
793 }
794 }
795 }
796 finally
797 {
798 cursor.close();
799 }
800 }
801
802 /**
803 * Iterate through the entries in ID2Subtree to perform a check for
804 * index cleanliness.
805 *
806 * @throws JebException If an error occurs in the JE backend.
807 * @throws DatabaseException If an error occurs in the JE database.
808 */
809 private void iterateID2Subtree() throws JebException, DatabaseException
810 {
811 Cursor cursor = id2s.openCursor(null, new CursorConfig());
812 try
813 {
814 DatabaseEntry key = new DatabaseEntry();
815 DatabaseEntry data = new DatabaseEntry();
816
817 OperationStatus status;
818 for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
819 status == OperationStatus.SUCCESS;
820 status = cursor.getNext(key, data, LockMode.DEFAULT))
821 {
822 keyCount++;
823
824 EntryID entryID;
825 try
826 {
827 entryID = new EntryID(key);
828 }
829 catch (Exception e)
830 {
831 errorCount++;
832 if (debugEnabled())
833 {
834 TRACER.debugCaught(DebugLogLevel.ERROR, e);
835
836 TRACER.debugError("File id2subtree has malformed ID %s%n",
837 StaticUtils.bytesToHex(key.getData()));
838 }
839 continue;
840 }
841
842 EntryIDSet entryIDList;
843 try
844 {
845 JebFormat.entryIDListFromDatabase(data.getData());
846 entryIDList = new EntryIDSet(key.getData(), data.getData());
847 }
848 catch (Exception e)
849 {
850 errorCount++;
851 if (debugEnabled())
852 {
853 TRACER.debugCaught(DebugLogLevel.ERROR, e);
854
855 TRACER.debugError("File id2subtree has malformed ID list " +
856 "for ID %s:%n%s%n", entryID,
857 StaticUtils.bytesToHex(data.getData()));
858 }
859 continue;
860 }
861
862 updateIndexStats(entryIDList);
863
864 if (entryIDList.isDefined())
865 {
866 Entry entry;
867 try
868 {
869 entry = id2entry.get(null, entryID, LockMode.DEFAULT);
870 }
871 catch (Exception e)
872 {
873 if (debugEnabled())
874 {
875 TRACER.debugCaught(DebugLogLevel.ERROR, e);
876 }
877 errorCount++;
878 continue;
879 }
880
881 if (entry == null)
882 {
883 errorCount++;
884 if (debugEnabled())
885 {
886 TRACER.debugError("File id2subtree has unknown ID %d%n",
887 entryID.longValue());
888 }
889 continue;
890 }
891
892 for (EntryID id : entryIDList)
893 {
894 Entry subordEntry;
895 try
896 {
897 subordEntry = id2entry.get(null, id, LockMode.DEFAULT);
898 }
899 catch (Exception e)
900 {
901 if (debugEnabled())
902 {
903 TRACER.debugCaught(DebugLogLevel.ERROR, e);
904 }
905 errorCount++;
906 continue;
907 }
908
909 if (subordEntry == null)
910 {
911 errorCount++;
912 if (debugEnabled())
913 {
914 TRACER.debugError("File id2subtree has ID %d referencing " +
915 "unknown ID %d%n", entryID.longValue(), id.longValue());
916 }
917 continue;
918 }
919
920 if (!subordEntry.getDN().isDescendantOf(entry.getDN()))
921 {
922 errorCount++;
923 if (debugEnabled())
924 {
925 TRACER.debugError("File id2subtree has ID %d with DN <%s> " +
926 "referencing ID %d with non-subordinate " +
927 "DN <%s>%n",
928 entryID.longValue(), entry.getDN().toString(),
929 id.longValue(), subordEntry.getDN().toString());
930 }
931 }
932 }
933 }
934 }
935 }
936 finally
937 {
938 cursor.close();
939 }
940 }
941
942 /**
943 * Increment the counter for a key that has exceeded the
944 * entry limit. The counter gives the number of entries that have
945 * referenced the key.
946 *
947 * @param index The index containing the key.
948 * @param key A key that has exceeded the entry limit.
949 */
950 private void incrEntryLimitStats(Index index, byte[] key)
951 {
952 HashMap<ByteString,Long> hashMap = entryLimitMap.get(index);
953 if (hashMap == null)
954 {
955 hashMap = new HashMap<ByteString, Long>();
956 entryLimitMap.put(index, hashMap);
957 }
958 ByteString octetString = new ASN1OctetString(key);
959 Long counter = hashMap.get(octetString);
960 if (counter == null)
961 {
962 counter = 1L;
963 }
964 else
965 {
966 counter++;
967 }
968 hashMap.put(octetString, counter);
969 }
970
971 /**
972 * Update the statistical information for an index record.
973 *
974 * @param entryIDSet The set of entry IDs for the index record.
975 */
976 private void updateIndexStats(EntryIDSet entryIDSet)
977 {
978 if (!entryIDSet.isDefined())
979 {
980 entryLimitExceededCount++;
981 multiReferenceCount++;
982 }
983 else
984 {
985 if (entryIDSet.size() > 1)
986 {
987 multiReferenceCount++;
988 }
989 entryReferencesCount += entryIDSet.size();
990 maxEntryPerValue = Math.max(maxEntryPerValue, entryIDSet.size());
991 }
992 }
993
994 /**
995 * Iterate through the entries in a VLV index to perform a check for index
996 * cleanliness.
997 *
998 * @param vlvIndex The VLV index to perform the check against.
999 * @param verifyID True to verify the IDs against id2entry.
1000 * @throws JebException If an error occurs in the JE backend.
1001 * @throws DatabaseException If an error occurs in the JE database.
1002 * @throws DirectoryException If an error occurs reading values in the index.
1003 */
1004 private void iterateVLVIndex(VLVIndex vlvIndex, boolean verifyID)
1005 throws JebException, DatabaseException, DirectoryException
1006 {
1007 if(vlvIndex == null)
1008 {
1009 return;
1010 }
1011
1012 Cursor cursor = vlvIndex.openCursor(null, new CursorConfig());
1013 try
1014 {
1015 DatabaseEntry key = new DatabaseEntry();
1016 OperationStatus status;
1017 LockMode lockMode = LockMode.DEFAULT;
1018 DatabaseEntry data = new DatabaseEntry();
1019
1020 status = cursor.getFirst(key, data, lockMode);
1021 SortValues lastValues = null;
1022 while(status == OperationStatus.SUCCESS)
1023 {
1024 SortValuesSet sortValuesSet =
1025 new SortValuesSet(key.getData(), data.getData(), vlvIndex);
1026 for(int i = 0; i < sortValuesSet.getEntryIDs().length; i++)
1027 {
1028 keyCount++;
1029 SortValues values = sortValuesSet.getSortValues(i);
1030 if(lastValues != null && lastValues.compareTo(values) >= 1)
1031 {
1032 // Make sure the values is larger then the previous one.
1033 if(debugEnabled())
1034 {
1035 TRACER.debugError("Values %s and %s are incorrectly ordered",
1036 lastValues, values, keyDump(vlvIndex,
1037 sortValuesSet.getKeySortValues()));
1038 }
1039 errorCount++;
1040 }
1041 if(i == sortValuesSet.getEntryIDs().length - 1 &&
1042 key.getData().length != 0)
1043 {
1044 // If this is the last one in a bounded set, make sure it is the
1045 // same as the database key.
1046 byte[] encodedKey = vlvIndex.encodeKey(values.getEntryID(),
1047 values.getValues());
1048 if(!Arrays.equals(key.getData(), encodedKey))
1049 {
1050 if(debugEnabled())
1051 {
1052 TRACER.debugError("Incorrect key for SortValuesSet in VLV " +
1053 "index %s. Last values bytes %s, Key bytes %s",
1054 vlvIndex.getName(), encodedKey, key);
1055 }
1056 errorCount++;
1057 }
1058 }
1059 lastValues = values;
1060
1061 if(verifyID)
1062 {
1063 Entry entry;
1064 EntryID id = new EntryID(values.getEntryID());
1065 try
1066 {
1067 entry = id2entry.get(null, id, LockMode.DEFAULT);
1068 }
1069 catch (Exception e)
1070 {
1071 if (debugEnabled())
1072 {
1073 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1074 }
1075 errorCount++;
1076 continue;
1077 }
1078
1079 if (entry == null)
1080 {
1081 errorCount++;
1082 if (debugEnabled())
1083 {
1084 TRACER.debugError("Reference to unknown ID %d%n%s",
1085 id.longValue(),
1086 keyDump(vlvIndex,
1087 sortValuesSet.getKeySortValues()));
1088 }
1089 continue;
1090 }
1091
1092 SortValues entryValues =
1093 new SortValues(id, entry, vlvIndex.sortOrder);
1094 if(entryValues.compareTo(values) != 0)
1095 {
1096 errorCount++;
1097 if(debugEnabled())
1098 {
1099 TRACER.debugError("Reference to entry ID %d " +
1100 "which does not match the values%n%s",
1101 id.longValue(),
1102 keyDump(vlvIndex,
1103 sortValuesSet.getKeySortValues()));
1104 }
1105 }
1106 }
1107 }
1108 status = cursor.getNext(key, data, lockMode);
1109 }
1110 }
1111 finally
1112 {
1113 cursor.close();
1114 }
1115 }
1116
1117 /**
1118 * Iterate through the entries in an attribute index to perform a check for
1119 * index cleanliness.
1120 * @param attrType The attribute type of the index to be checked.
1121 * @param index The index database to be checked.
1122 * @param indexType Type of the index (ie, SUBSTRING, ORDERING)
1123 * @throws JebException If an error occurs in the JE backend.
1124 * @throws DatabaseException If an error occurs in the JE database.
1125 */
1126 private void iterateAttrIndex(AttributeType attrType,
1127 Index index, IndexType indexType)
1128 throws JebException, DatabaseException
1129 {
1130 if (index == null)
1131 {
1132 return;
1133 }
1134
1135 Cursor cursor = index.openCursor(null, new CursorConfig());
1136 try
1137 {
1138 DatabaseEntry key = new DatabaseEntry();
1139 DatabaseEntry data = new DatabaseEntry();
1140
1141 OrderingMatchingRule orderingMatchingRule =
1142 attrType.getOrderingMatchingRule();
1143 ApproximateMatchingRule approximateMatchingRule =
1144 attrType.getApproximateMatchingRule();
1145 ASN1OctetString previousValue = null;
1146
1147 OperationStatus status;
1148 for (status = cursor.getFirst(key, data, LockMode.DEFAULT);
1149 status == OperationStatus.SUCCESS;
1150 status = cursor.getNext(key, data, LockMode.DEFAULT))
1151 {
1152 keyCount++;
1153
1154 EntryIDSet entryIDList;
1155 try
1156 {
1157 JebFormat.entryIDListFromDatabase(data.getData());
1158 entryIDList = new EntryIDSet(key.getData(), data.getData());
1159 }
1160 catch (Exception e)
1161 {
1162 errorCount++;
1163 if (debugEnabled())
1164 {
1165 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1166
1167 TRACER.debugError("Malformed ID list: %s%n%s",
1168 StaticUtils.bytesToHex(data.getData()),
1169 keyDump(index, key.getData()));
1170 }
1171 continue;
1172 }
1173
1174 updateIndexStats(entryIDList);
1175
1176 if (entryIDList.isDefined())
1177 {
1178 byte[] value = key.getData();
1179 SearchFilter sf;
1180 AttributeValue assertionValue;
1181
1182 switch (indexType)
1183 {
1184 case SUBSTRING:
1185 ArrayList<ByteString> subAnyElements =
1186 new ArrayList<ByteString>(1);
1187 subAnyElements.add(new ASN1OctetString(value));
1188
1189 sf = SearchFilter.createSubstringFilter(attrType,null,
1190 subAnyElements,null);
1191 break;
1192 case ORDERING:
1193 // Ordering index checking is two fold:
1194 // 1. Make sure the entry has an attribute value that is the same
1195 // as the key. This is done by falling through to the next
1196 // case and create an equality filter.
1197 // 2. Make sure the key value is greater then the previous key
1198 // value.
1199 assertionValue =
1200 new AttributeValue(attrType, new ASN1OctetString(value));
1201
1202 sf = SearchFilter.createEqualityFilter(attrType,assertionValue);
1203
1204 if(orderingMatchingRule != null && previousValue != null)
1205 {
1206 ASN1OctetString thisValue = new ASN1OctetString(value);
1207 int order = orderingMatchingRule.compareValues(thisValue,
1208 previousValue);
1209 if(order > 0)
1210 {
1211 errorCount++;
1212 if(debugEnabled())
1213 {
1214 TRACER.debugError("Reversed ordering of index keys " +
1215 "(keys dumped in the order found in database)%n" +
1216 "Key 1:%n%s%nKey 2:%n%s",
1217 keyDump(index, thisValue.value()),
1218 keyDump(index,previousValue.value()));
1219 }
1220 continue;
1221 }
1222 else if(order == 0)
1223 {
1224 errorCount++;
1225 if(debugEnabled())
1226 {
1227 TRACER.debugError("Duplicate index keys%nKey 1:%n%s%n" +
1228 "Key2:%n%s", keyDump(index, thisValue.value()),
1229 keyDump(index,previousValue.value()));
1230 }
1231 continue;
1232 }
1233 else
1234 {
1235 previousValue = thisValue;
1236 }
1237 }
1238 break;
1239 case EQ:
1240 assertionValue =
1241 new AttributeValue(attrType, new ASN1OctetString(value));
1242
1243 sf = SearchFilter.createEqualityFilter(attrType,assertionValue);
1244 break;
1245
1246 case PRES:
1247 sf = SearchFilter.createPresenceFilter(attrType);
1248 break;
1249
1250 case APPROXIMATE:
1251 // This must be handled differently since we can't use a search
1252 // filter to see if the key matches.
1253 sf = null;
1254 break;
1255
1256 default:
1257 errorCount++;
1258 if (debugEnabled())
1259 {
1260 TRACER.debugError("Malformed value%n%s", keyDump(index, value));
1261 }
1262 continue;
1263 }
1264
1265 EntryID prevID = null;
1266 for (EntryID id : entryIDList)
1267 {
1268 if (prevID != null && id.equals(prevID))
1269 {
1270 if (debugEnabled())
1271 {
1272 TRACER.debugError("Duplicate reference to ID %d%n%s",
1273 id.longValue(), keyDump(index, key.getData()));
1274 }
1275 }
1276 prevID = id;
1277
1278 Entry entry;
1279 try
1280 {
1281 entry = id2entry.get(null, id, LockMode.DEFAULT);
1282 }
1283 catch (Exception e)
1284 {
1285 if (debugEnabled())
1286 {
1287 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1288 }
1289 errorCount++;
1290 continue;
1291 }
1292
1293 if (entry == null)
1294 {
1295 errorCount++;
1296 if (debugEnabled())
1297 {
1298 TRACER.debugError("Reference to unknown ID %d%n%s",
1299 id.longValue(), keyDump(index, key.getData()));
1300 }
1301 continue;
1302 }
1303
1304 try
1305 {
1306 boolean match = false;
1307 if(indexType != IndexType.APPROXIMATE)
1308 {
1309 match = sf.matchesEntry(entry);
1310 }
1311 else
1312 {
1313 ByteString normalizedValue = new ASN1OctetString(value);
1314 List<Attribute> attrs = entry.getAttribute(attrType);
1315 if ((attrs != null) && (!attrs.isEmpty()))
1316 {
1317 for (Attribute a : attrs)
1318 {
1319 for (AttributeValue v : a.getValues())
1320 {
1321 ByteString nv =
1322 approximateMatchingRule.normalizeValue(v.getValue());
1323 match = approximateMatchingRule.
1324 approximatelyMatch(nv, normalizedValue);
1325 if(match)
1326 {
1327 break;
1328 }
1329 }
1330 if(match)
1331 {
1332 break;
1333 }
1334 }
1335 }
1336 }
1337
1338 if (!match)
1339 {
1340 errorCount++;
1341 if (debugEnabled())
1342 {
1343 TRACER.debugError("Reference to entry " +
1344 "<%s> which does not match the value%n%s",
1345 entry.getDN(), keyDump(index, value));
1346 }
1347 }
1348 }
1349 catch (DirectoryException e)
1350 {
1351 if (debugEnabled())
1352 {
1353 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1354 }
1355 }
1356 }
1357 }
1358 }
1359 }
1360 finally
1361 {
1362 cursor.close();
1363 }
1364 }
1365
1366 /**
1367 * Check that an index is complete for a given entry.
1368 *
1369 * @param entryID The entry ID.
1370 * @param entry The entry to be checked.
1371 */
1372 private void verifyEntry(EntryID entryID, Entry entry)
1373 {
1374 if (verifyDN2ID)
1375 {
1376 verifyDN2ID(entryID, entry);
1377 }
1378 if (verifyID2Children)
1379 {
1380 verifyID2Children(entryID, entry);
1381 }
1382 if (verifyID2Subtree)
1383 {
1384 verifyID2Subtree(entryID, entry);
1385 }
1386 verifyIndex(entryID, entry);
1387 }
1388
1389 /**
1390 * Check that the DN2ID index is complete for a given entry.
1391 *
1392 * @param entryID The entry ID.
1393 * @param entry The entry to be checked.
1394 */
1395 private void verifyDN2ID(EntryID entryID, Entry entry)
1396 {
1397 DN dn = entry.getDN();
1398
1399 // Check the ID is in dn2id with the correct DN.
1400 try
1401 {
1402 EntryID id = dn2id.get(null, dn, LockMode.DEFAULT);
1403 if (id == null)
1404 {
1405 if (debugEnabled())
1406 {
1407 TRACER.debugError("File dn2id is missing key %s.%n",
1408 dn.toNormalizedString());
1409 }
1410 errorCount++;
1411 }
1412 else if (!id.equals(entryID))
1413 {
1414 if (debugEnabled())
1415 {
1416 TRACER.debugError("File dn2id has ID %d instead of %d for key %s.%n",
1417 id.longValue(),
1418 entryID.longValue(),
1419 dn.toNormalizedString());
1420 }
1421 errorCount++;
1422 }
1423 }
1424 catch (Exception e)
1425 {
1426 if (debugEnabled())
1427 {
1428 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1429
1430 TRACER.debugError("File dn2id has error reading key %s: %s.%n",
1431 dn.toNormalizedString(),
1432 e.getMessage());
1433 }
1434 errorCount++;
1435 }
1436
1437 // Check the parent DN is in dn2id.
1438 DN parentDN = getParent(dn);
1439 if (parentDN != null)
1440 {
1441 try
1442 {
1443 EntryID id = dn2id.get(null, parentDN, LockMode.DEFAULT);
1444 if (id == null)
1445 {
1446 if (debugEnabled())
1447 {
1448 TRACER.debugError("File dn2id is missing key %s.%n",
1449 parentDN.toNormalizedString());
1450 }
1451 errorCount++;
1452 }
1453 }
1454 catch (Exception e)
1455 {
1456 if (debugEnabled())
1457 {
1458 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1459
1460 TRACER.debugError("File dn2id has error reading key %s: %s.%n",
1461 parentDN.toNormalizedString(),
1462 e.getMessage());
1463 }
1464 errorCount++;
1465 }
1466 }
1467 }
1468
1469 /**
1470 * Check that the ID2Children index is complete for a given entry.
1471 *
1472 * @param entryID The entry ID.
1473 * @param entry The entry to be checked.
1474 */
1475 private void verifyID2Children(EntryID entryID, Entry entry)
1476 {
1477 DN dn = entry.getDN();
1478
1479 DN parentDN = getParent(dn);
1480 if (parentDN != null)
1481 {
1482 EntryID parentID = null;
1483 try
1484 {
1485 parentID = dn2id.get(null, parentDN, LockMode.DEFAULT);
1486 if (parentID == null)
1487 {
1488 if (debugEnabled())
1489 {
1490 TRACER.debugError("File dn2id is missing key %s.%n",
1491 parentDN.toNormalizedString());
1492 }
1493 errorCount++;
1494 }
1495 }
1496 catch (Exception e)
1497 {
1498 if (debugEnabled())
1499 {
1500 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1501
1502 TRACER.debugError("File dn2id has error reading key %s: %s.",
1503 parentDN.toNormalizedString(),
1504 e.getMessage());
1505 }
1506 errorCount++;
1507 }
1508 if (parentID != null)
1509 {
1510 try
1511 {
1512 ConditionResult cr;
1513 cr = id2c.containsID(null, parentID.getDatabaseEntry(), entryID);
1514 if (cr == ConditionResult.FALSE)
1515 {
1516 if (debugEnabled())
1517 {
1518 TRACER.debugError("File id2children is missing ID %d " +
1519 "for key %d.%n",
1520 entryID.longValue(), parentID.longValue());
1521 }
1522 errorCount++;
1523 }
1524 else if (cr == ConditionResult.UNDEFINED)
1525 {
1526 incrEntryLimitStats(id2c, parentID.getDatabaseEntry().getData());
1527 }
1528 }
1529 catch (DatabaseException e)
1530 {
1531 if (debugEnabled())
1532 {
1533 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1534
1535 TRACER.debugError("File id2children has error reading key %d: %s.",
1536 parentID.longValue(), e.getMessage());
1537 }
1538 errorCount++;
1539 }
1540 }
1541 }
1542 }
1543
1544 /**
1545 * Check that the ID2Subtree index is complete for a given entry.
1546 *
1547 * @param entryID The entry ID.
1548 * @param entry The entry to be checked.
1549 */
1550 private void verifyID2Subtree(EntryID entryID, Entry entry)
1551 {
1552 for (DN dn = getParent(entry.getDN()); dn != null; dn = getParent(dn))
1553 {
1554 EntryID id = null;
1555 try
1556 {
1557 id = dn2id.get(null, dn, LockMode.DEFAULT);
1558 if (id == null)
1559 {
1560 if (debugEnabled())
1561 {
1562 TRACER.debugError("File dn2id is missing key %s.%n",
1563 dn.toNormalizedString());
1564 }
1565 errorCount++;
1566 }
1567 }
1568 catch (Exception e)
1569 {
1570 if (debugEnabled())
1571 {
1572 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1573
1574 TRACER.debugError("File dn2id has error reading key %s: %s.%n",
1575 dn.toNormalizedString(),
1576 e.getMessage());
1577 }
1578 errorCount++;
1579 }
1580 if (id != null)
1581 {
1582 try
1583 {
1584 ConditionResult cr;
1585 cr = id2s.containsID(null, id.getDatabaseEntry(), entryID);
1586 if (cr == ConditionResult.FALSE)
1587 {
1588 if (debugEnabled())
1589 {
1590 TRACER.debugError("File id2subtree is missing ID %d " +
1591 "for key %d.%n",
1592 entryID.longValue(), id.longValue());
1593 }
1594 errorCount++;
1595 }
1596 else if (cr == ConditionResult.UNDEFINED)
1597 {
1598 incrEntryLimitStats(id2s, id.getDatabaseEntry().getData());
1599 }
1600 }
1601 catch (DatabaseException e)
1602 {
1603 if (debugEnabled())
1604 {
1605 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1606
1607 TRACER.debugError("File id2subtree has error reading key %d: %s.%n",
1608 id.longValue(), e.getMessage());
1609 }
1610 errorCount++;
1611 }
1612 }
1613 }
1614 }
1615
1616 /**
1617 * Construct a printable string from a raw key value.
1618 *
1619 * @param index The index database containing the key value.
1620 * @param keyBytes The bytes of the key.
1621 * @return A string that may be logged or printed.
1622 */
1623 private String keyDump(Index index, byte[] keyBytes)
1624 {
1625 /*
1626 String str;
1627 try
1628 {
1629 str = new String(keyBytes, "UTF-8");
1630 }
1631 catch (UnsupportedEncodingException e)
1632 {
1633 str = StaticUtils.bytesToHex(keyBytes);
1634 }
1635 return str;
1636 */
1637 StringBuilder buffer = new StringBuilder(128);
1638 buffer.append("File: ");
1639 buffer.append(index.toString());
1640 buffer.append(ServerConstants.EOL);
1641 buffer.append("Key:");
1642 buffer.append(ServerConstants.EOL);
1643 StaticUtils.byteArrayToHexPlusAscii(buffer, keyBytes, 6);
1644 return buffer.toString();
1645 }
1646
1647 /**
1648 * Construct a printable string from a raw key value.
1649 *
1650 * @param vlvIndex The vlvIndex database containing the key value.
1651 * @param keySortValues THe sort values that is being used as the key.
1652 * @return A string that may be logged or printed.
1653 */
1654 private String keyDump(VLVIndex vlvIndex, SortValues keySortValues)
1655 {
1656 /*
1657 String str;
1658 try
1659 {
1660 str = new String(keyBytes, "UTF-8");
1661 }
1662 catch (UnsupportedEncodingException e)
1663 {
1664 str = StaticUtils.bytesToHex(keyBytes);
1665 }
1666 return str;
1667 */
1668 StringBuilder buffer = new StringBuilder(128);
1669 buffer.append("File: ");
1670 buffer.append(vlvIndex.toString());
1671 buffer.append(ServerConstants.EOL);
1672 buffer.append("Key (last sort values):");
1673 if(keySortValues == null)
1674 {
1675 buffer.append("UNBOUNDED (0x00)");
1676 }
1677 else
1678 {
1679 buffer.append(keySortValues.toString());
1680 }
1681 return buffer.toString();
1682 }
1683
1684 /**
1685 * Check that an attribute index is complete for a given entry.
1686 *
1687 * @param entryID The entry ID.
1688 * @param entry The entry to be checked.
1689 */
1690 private void verifyIndex(EntryID entryID, Entry entry)
1691 {
1692 for (AttributeIndex attrIndex : attrIndexList)
1693 {
1694 try
1695 {
1696 List<Attribute> attrList =
1697 entry.getAttribute(attrIndex.getAttributeType());
1698 if (attrList != null)
1699 {
1700 verifyAttribute(attrIndex, entryID, attrList);
1701 }
1702 }
1703 catch (DirectoryException e)
1704 {
1705 if (debugEnabled())
1706 {
1707 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1708
1709 TRACER.debugError("Error normalizing values of attribute %s in " +
1710 "entry <%s>: %s.%n",
1711 attrIndex.getAttributeType().toString(),
1712 entry.getDN().toString(),
1713 String.valueOf(e.getMessageObject()));
1714 }
1715 }
1716 }
1717
1718 for (VLVIndex vlvIndex : vlvIndexList)
1719 {
1720 try
1721 {
1722 if(vlvIndex.shouldInclude(entry))
1723 {
1724 if(!vlvIndex.containsValues(null, entryID.longValue(),
1725 vlvIndex.getSortValues(entry)))
1726 {
1727 if(debugEnabled())
1728 {
1729 TRACER.debugError("Missing entry %s in VLV index %s",
1730 entry.getDN().toString(),
1731 vlvIndex.getName());
1732 }
1733 errorCount++;
1734 }
1735 }
1736 }
1737 catch (DirectoryException e)
1738 {
1739 if (debugEnabled())
1740 {
1741 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1742
1743 TRACER.debugError("Error checking entry %s against filter or " +
1744 "base DN for VLV index %s: %s",
1745 entry.getDN().toString(),
1746 vlvIndex.getName(),
1747 String.valueOf(e.getMessageObject()));
1748 }
1749 errorCount++;
1750 }
1751 catch (DatabaseException e)
1752 {
1753 if (debugEnabled())
1754 {
1755 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1756
1757 TRACER.debugError("Error reading VLV index %s for entry %s: %s",
1758 vlvIndex.getName(),
1759 entry.getDN().toString(),
1760 StaticUtils.getBacktrace(e));
1761 }
1762 errorCount++;
1763 }
1764 catch (JebException e)
1765 {
1766 if (debugEnabled())
1767 {
1768 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1769
1770 TRACER.debugError("Error reading VLV index %s for entry %s: %s",
1771 vlvIndex.getName(),
1772 entry.getDN().toString(),
1773 StaticUtils.getBacktrace(e));
1774 }
1775 errorCount++;
1776 }
1777 }
1778 }
1779
1780 /**
1781 * Check that an attribute index is complete for a given attribute.
1782 *
1783 * @param attrIndex The attribute index to be checked.
1784 * @param entryID The entry ID.
1785 * @param attrList The attribute to be checked.
1786 * @throws DirectoryException If a Directory Server error occurs.
1787 */
1788 private void verifyAttribute(AttributeIndex attrIndex, EntryID entryID,
1789 List<Attribute> attrList)
1790 throws DirectoryException
1791 {
1792 Transaction txn = null;
1793 Index equalityIndex = attrIndex.equalityIndex;
1794 Index presenceIndex = attrIndex.presenceIndex;
1795 Index substringIndex = attrIndex.substringIndex;
1796 Index orderingIndex = attrIndex.orderingIndex;
1797 Index approximateIndex = attrIndex.approximateIndex;
1798 DatabaseEntry presenceKey = AttributeIndex.presenceKey;
1799
1800 // Presence index.
1801 if (!attrList.isEmpty() && presenceIndex != null)
1802 {
1803 try
1804 {
1805 ConditionResult cr;
1806 cr = presenceIndex.containsID(txn, presenceKey, entryID);
1807 if (cr == ConditionResult.FALSE)
1808 {
1809 if (debugEnabled())
1810 {
1811 TRACER.debugError("Missing ID %d%n%s",
1812 entryID.longValue(),
1813 keyDump(presenceIndex, presenceKey.getData()));
1814 }
1815 errorCount++;
1816 }
1817 else if (cr == ConditionResult.UNDEFINED)
1818 {
1819 incrEntryLimitStats(presenceIndex, presenceKey.getData());
1820 }
1821 }
1822 catch (DatabaseException e)
1823 {
1824 if (debugEnabled())
1825 {
1826 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1827
1828 TRACER.debugError("Error reading database: %s%n%s",
1829 e.getMessage(),
1830 keyDump(presenceIndex, presenceKey.getData()));
1831 }
1832 errorCount++;
1833 }
1834 }
1835
1836 if (attrList != null)
1837 {
1838 for (Attribute attr : attrList)
1839 {
1840 LinkedHashSet<AttributeValue> values = attr.getValues();
1841 for (AttributeValue value : values)
1842 {
1843 byte[] normalizedBytes = value.getNormalizedValue().value();
1844
1845 // Equality index.
1846 if (equalityIndex != null)
1847 {
1848 DatabaseEntry key = new DatabaseEntry(normalizedBytes);
1849 try
1850 {
1851 ConditionResult cr;
1852 cr = equalityIndex.containsID(txn, key, entryID);
1853 if (cr == ConditionResult.FALSE)
1854 {
1855 if (debugEnabled())
1856 {
1857 TRACER.debugError("Missing ID %d%n%s",
1858 entryID.longValue(),
1859 keyDump(equalityIndex, normalizedBytes));
1860 }
1861 errorCount++;
1862 }
1863 else if (cr == ConditionResult.UNDEFINED)
1864 {
1865 incrEntryLimitStats(equalityIndex, normalizedBytes);
1866 }
1867 }
1868 catch (DatabaseException e)
1869 {
1870 if (debugEnabled())
1871 {
1872 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1873
1874 TRACER.debugError("Error reading database: %s%n%s",
1875 e.getMessage(),
1876 keyDump(equalityIndex, normalizedBytes));
1877 }
1878 errorCount++;
1879 }
1880 }
1881
1882 // Substring index.
1883 if (substringIndex != null)
1884 {
1885 Set<ByteString> keyBytesSet =
1886 attrIndex.substringKeys(normalizedBytes);
1887 DatabaseEntry key = new DatabaseEntry();
1888 for (ByteString keyBytes : keyBytesSet)
1889 {
1890 key.setData(keyBytes.value());
1891 try
1892 {
1893 ConditionResult cr;
1894 cr = substringIndex.containsID(txn, key, entryID);
1895 if (cr == ConditionResult.FALSE)
1896 {
1897 if (debugEnabled())
1898 {
1899 TRACER.debugError("Missing ID %d%n%s",
1900 entryID.longValue(),
1901 keyDump(substringIndex, key.getData()));
1902 }
1903 errorCount++;
1904 }
1905 else if (cr == ConditionResult.UNDEFINED)
1906 {
1907 incrEntryLimitStats(substringIndex, key.getData());
1908 }
1909 }
1910 catch (DatabaseException e)
1911 {
1912 if (debugEnabled())
1913 {
1914 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1915
1916 TRACER.debugError("Error reading database: %s%n%s",
1917 e.getMessage(),
1918 keyDump(substringIndex, key.getData()));
1919 }
1920 errorCount++;
1921 }
1922 }
1923 }
1924
1925 // Ordering index.
1926 if (orderingIndex != null)
1927 {
1928 // Use the ordering matching rule to normalize the value.
1929 OrderingMatchingRule orderingRule =
1930 attr.getAttributeType().getOrderingMatchingRule();
1931
1932 normalizedBytes =
1933 orderingRule.normalizeValue(value.getValue()).value();
1934
1935 DatabaseEntry key = new DatabaseEntry(normalizedBytes);
1936 try
1937 {
1938 ConditionResult cr;
1939 cr = orderingIndex.containsID(txn, key, entryID);
1940 if (cr == ConditionResult.FALSE)
1941 {
1942 if (debugEnabled())
1943 {
1944 TRACER.debugError("Missing ID %d%n%s",
1945 entryID.longValue(),
1946 keyDump(orderingIndex, normalizedBytes));
1947 }
1948 errorCount++;
1949 }
1950 else if (cr == ConditionResult.UNDEFINED)
1951 {
1952 incrEntryLimitStats(orderingIndex, normalizedBytes);
1953 }
1954 }
1955 catch (DatabaseException e)
1956 {
1957 if (debugEnabled())
1958 {
1959 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1960
1961 TRACER.debugError("Error reading database: %s%n%s",
1962 e.getMessage(),
1963 keyDump(orderingIndex, normalizedBytes));
1964 }
1965 errorCount++;
1966 }
1967 }
1968 // Approximate index.
1969 if (approximateIndex != null)
1970 {
1971 // Use the approximate matching rule to normalize the value.
1972 ApproximateMatchingRule approximateRule =
1973 attr.getAttributeType().getApproximateMatchingRule();
1974
1975 normalizedBytes =
1976 approximateRule.normalizeValue(value.getValue()).value();
1977
1978 DatabaseEntry key = new DatabaseEntry(normalizedBytes);
1979 try
1980 {
1981 ConditionResult cr;
1982 cr = approximateIndex.containsID(txn, key, entryID);
1983 if (cr == ConditionResult.FALSE)
1984 {
1985 if (debugEnabled())
1986 {
1987 TRACER.debugError("Missing ID %d%n%s",
1988 entryID.longValue(),
1989 keyDump(orderingIndex, normalizedBytes));
1990 }
1991 errorCount++;
1992 }
1993 else if (cr == ConditionResult.UNDEFINED)
1994 {
1995 incrEntryLimitStats(orderingIndex, normalizedBytes);
1996 }
1997 }
1998 catch (DatabaseException e)
1999 {
2000 if (debugEnabled())
2001 {
2002 TRACER.debugCaught(DebugLogLevel.ERROR, e);
2003
2004 TRACER.debugError("Error reading database: %s%n%s",
2005 e.getMessage(),
2006 keyDump(approximateIndex, normalizedBytes));
2007 }
2008 errorCount++;
2009 }
2010 }
2011 }
2012 }
2013 }
2014 }
2015
2016 /**
2017 * Get the parent DN of a given DN.
2018 *
2019 * @param dn The DN.
2020 * @return The parent DN or null if the given DN is a base DN.
2021 */
2022 private DN getParent(DN dn)
2023 {
2024 if (dn.equals(verifyConfig.getBaseDN()))
2025 {
2026 return null;
2027 }
2028 return dn.getParentDNInSuffix();
2029 }
2030
2031 /**
2032 * This class reports progress of the verify job at fixed intervals.
2033 */
2034 class ProgressTask extends TimerTask
2035 {
2036 /**
2037 * The total number of records to process.
2038 */
2039 private long totalCount;
2040
2041 /**
2042 * The number of records that had been processed at the time of the
2043 * previous progress report.
2044 */
2045 private long previousCount = 0;
2046
2047 /**
2048 * The time in milliseconds of the previous progress report.
2049 */
2050 private long previousTime;
2051
2052 /**
2053 * The environment statistics at the time of the previous report.
2054 */
2055 private EnvironmentStats prevEnvStats;
2056
2057 /**
2058 * The number of bytes in a megabyte.
2059 * Note that 1024*1024 bytes may eventually become known as a mebibyte(MiB).
2060 */
2061 private static final int bytesPerMegabyte = 1024*1024;
2062
2063 /**
2064 * Create a new verify progress task.
2065 * @throws DatabaseException An error occurred while accessing the JE
2066 * database.
2067 */
2068 public ProgressTask() throws DatabaseException
2069 {
2070 previousTime = System.currentTimeMillis();
2071 prevEnvStats =
2072 rootContainer.getEnvironmentStats(new StatsConfig());
2073 totalCount = rootContainer.getEntryContainer(
2074 verifyConfig.getBaseDN()).getEntryCount();
2075 }
2076
2077 /**
2078 * The action to be performed by this timer task.
2079 */
2080 public void run()
2081 {
2082 long latestCount = keyCount;
2083 long deltaCount = (latestCount - previousCount);
2084 long latestTime = System.currentTimeMillis();
2085 long deltaTime = latestTime - previousTime;
2086
2087 if (deltaTime == 0)
2088 {
2089 return;
2090 }
2091
2092 float rate = 1000f*deltaCount / deltaTime;
2093
2094 Message message = NOTE_JEB_VERIFY_PROGRESS_REPORT.get(
2095 latestCount, totalCount, errorCount, rate);
2096 logError(message);
2097
2098 try
2099 {
2100 Runtime runtime = Runtime.getRuntime();
2101 long freeMemory = runtime.freeMemory() / bytesPerMegabyte;
2102
2103 EnvironmentStats envStats =
2104 rootContainer.getEnvironmentStats(new StatsConfig());
2105 long nCacheMiss =
2106 envStats.getNCacheMiss() - prevEnvStats.getNCacheMiss();
2107
2108 float cacheMissRate = 0;
2109 if (deltaCount > 0)
2110 {
2111 cacheMissRate = nCacheMiss/(float)deltaCount;
2112 }
2113
2114 message = INFO_JEB_VERIFY_CACHE_AND_MEMORY_REPORT.get(
2115 freeMemory, cacheMissRate);
2116 logError(message);
2117
2118 prevEnvStats = envStats;
2119 }
2120 catch (DatabaseException e)
2121 {
2122 if (debugEnabled())
2123 {
2124 TRACER.debugCaught(DebugLogLevel.ERROR, e);
2125 }
2126 }
2127
2128
2129 previousCount = latestCount;
2130 previousTime = latestTime;
2131 }
2132 }
2133
2134 /**
2135 * Adds an attribute of type t and value v to the statEntry, only if the
2136 * statEntry is not null.
2137 * @param statEntry passed in from backentryImpl.verifyBackend.
2138 * @param t String to be used as the attribute type.
2139 * @param v String to be used as the attribute value.
2140 */
2141 private void addStatEntry(Entry statEntry, String t, String v)
2142 {
2143 if (statEntry != null)
2144 {
2145 Attribute a = new Attribute(t, v);
2146 statEntry.addAttribute(a, null);
2147 }
2148 }
2149 }