001 /*
002 * CDDL HEADER START
003 *
004 * The contents of this file are subject to the terms of the
005 * Common Development and Distribution License, Version 1.0 only
006 * (the "License"). You may not use this file except in compliance
007 * with the License.
008 *
009 * You can obtain a copy of the license at
010 * trunk/opends/resource/legal-notices/OpenDS.LICENSE
011 * or https://OpenDS.dev.java.net/OpenDS.LICENSE.
012 * See the License for the specific language governing permissions
013 * and limitations under the License.
014 *
015 * When distributing Covered Code, include this CDDL HEADER in each
016 * file and include the License file at
017 * trunk/opends/resource/legal-notices/OpenDS.LICENSE. If applicable,
018 * add the following below this CDDL HEADER, with the fields enclosed
019 * by brackets "[]" replaced with your own identifying information:
020 * Portions Copyright [yyyy] [name of copyright owner]
021 *
022 * CDDL HEADER END
023 *
024 *
025 * Copyright 2006-2008 Sun Microsystems, Inc.
026 */
027 package org.opends.server.backends.jeb;
028
029 import static org.opends.server.loggers.debug.DebugLogger.*;
030 import org.opends.server.loggers.debug.DebugTracer;
031 import static org.opends.server.loggers.ErrorLogger.*;
032
033 import com.sleepycat.je.*;
034
035 import org.opends.server.types.*;
036 import org.opends.server.util.StaticUtils;
037 import org.opends.server.backends.jeb.importLDIF.IntegerImportIDSet;
038 import org.opends.server.backends.jeb.importLDIF.ImportIDSet;
039 import static org.opends.messages.JebMessages.*;
040
041 import java.util.*;
042
043 /**
044 * Represents an index implemented by a JE database in which each key maps to
045 * a set of entry IDs. The key is a byte array, and is constructed from some
046 * normalized form of an attribute value (or fragment of a value) appearing
047 * in the entry.
048 */
049 public class Index extends DatabaseContainer
050 {
051 /**
052 * The tracer object for the debug logger.
053 */
054 private static final DebugTracer TRACER = getTracer();
055
056 /**
057 * The indexer object to construct index keys from LDAP attribute values.
058 */
059 public Indexer indexer;
060
061 /**
062 * The comparator for index keys.
063 */
064 private Comparator<byte[]> comparator;
065
066 /**
067 * The limit on the number of entry IDs that may be indexed by one key.
068 */
069 private int indexEntryLimit;
070
071 /**
072 * Limit on the number of entry IDs that may be retrieved by cursoring
073 * through an index.
074 */
075 private int cursorEntryLimit;
076
077 /**
078 * Number of keys that have exceeded the entry limit since this
079 * object was created.
080 */
081 private int entryLimitExceededCount;
082
083 /**
084 * The max number of tries to rewrite phantom records.
085 */
086 final int phantomWriteRetires = 3;
087
088 /**
089 * Whether to maintain a count of IDs for a key once the entry limit
090 * has exceeded.
091 */
092 boolean maintainCount;
093
094 private State state;
095
096 /**
097 * A flag to indicate if this index should be trusted to be consistent
098 * with the entries database. If not trusted, we assume that existing
099 * entryIDSets for a key is still accurate. However, keys that do not
100 * exist are undefined instead of an empty entryIDSet. The following
101 * rules will be observed when the index is not trusted:
102 *
103 * - no entryIDs will be added to a non-existing key.
104 * - undefined entryIdSet will be returned whenever a key is not found.
105 */
106 private boolean trusted = false;
107
108 /**
109 * A flag to indicate if a rebuild process is running on this index.
110 * During the rebuild process, we assume that no entryIDSets are
111 * accurate and return an undefined set on all read operations.
112 * However all write opeations will succeed. The rebuildRunning
113 * flag overrides all behaviours of the trusted flag.
114 */
115 private boolean rebuildRunning = false;
116
117
118 /**
119 * Create a new index object.
120 * @param name The name of the index database within the entryContainer.
121 * @param indexer The indexer object to construct index keys from LDAP
122 * attribute values.
123 * @param state The state database to persist index state info.
124 * @param indexEntryLimit The configured limit on the number of entry IDs
125 * that may be indexed by one key.
126 * @param cursorEntryLimit The configured limit on the number of entry IDs
127 * @param maintainCount Whether to maintain a count of IDs for a key once
128 * the entry limit has exceeded.
129 * @param env The JE Environemnt
130 * @param entryContainer The database entryContainer holding this index.
131 * @throws DatabaseException If an error occurs in the JE database.
132 */
133 public Index(String name, Indexer indexer, State state,
134 int indexEntryLimit, int cursorEntryLimit, boolean maintainCount,
135 Environment env, EntryContainer entryContainer)
136 throws DatabaseException
137 {
138 super(name, env, entryContainer);
139 this.indexer = indexer;
140 this.comparator = indexer.getComparator();
141 this.indexEntryLimit = indexEntryLimit;
142 this.cursorEntryLimit = cursorEntryLimit;
143 this.maintainCount = maintainCount;
144
145 DatabaseConfig dbNodupsConfig = new DatabaseConfig();
146
147 if(env.getConfig().getReadOnly())
148 {
149 dbNodupsConfig.setReadOnly(true);
150 dbNodupsConfig.setAllowCreate(false);
151 dbNodupsConfig.setTransactional(false);
152 }
153 else if(!env.getConfig().getTransactional())
154 {
155 dbNodupsConfig.setAllowCreate(true);
156 dbNodupsConfig.setTransactional(false);
157 dbNodupsConfig.setDeferredWrite(true);
158 }
159 else
160 {
161 dbNodupsConfig.setAllowCreate(true);
162 dbNodupsConfig.setTransactional(true);
163 }
164
165 this.dbConfig = dbNodupsConfig;
166 this.dbConfig.setOverrideBtreeComparator(true);
167 this.dbConfig.setBtreeComparator(comparator.getClass());
168
169 this.state = state;
170
171 this.trusted = state.getIndexTrustState(null, this);
172 if(!trusted && entryContainer.getHighestEntryID().equals(new EntryID(0)))
173 {
174 // If there are no entries in the entry container then there
175 // is no reason why this index can't be upgraded to trusted.
176 setTrusted(null, true);
177 }
178
179 // Issue warning if this index is not trusted
180 if(!trusted)
181 {
182 logError(NOTE_JEB_INDEX_ADD_REQUIRES_REBUILD.get(name));
183 }
184
185 }
186
187 /**
188 * Add an add entry ID operation into a index buffer.
189 *
190 * @param buffer The index buffer to insert the ID into.
191 * @param keyBytes The index key bytes.
192 * @param entryID The entry ID.
193 * @return True if the entry ID is inserted or ignored because the entry limit
194 * count is exceeded. False if it already exists in the entry ID set
195 * for the given key.
196 */
197 public boolean insertID(IndexBuffer buffer, byte[] keyBytes,
198 EntryID entryID)
199 {
200 TreeMap<byte[], IndexBuffer.BufferedIndexValues> bufferedOperations =
201 buffer.getBufferedIndex(this);
202 IndexBuffer.BufferedIndexValues values = null;
203
204 if(bufferedOperations == null)
205 {
206 bufferedOperations = new TreeMap<byte[],
207 IndexBuffer.BufferedIndexValues>(comparator);
208 buffer.putBufferedIndex(this, bufferedOperations);
209 }
210 else
211 {
212 values = bufferedOperations.get(keyBytes);
213 }
214
215 if(values == null)
216 {
217 values = new IndexBuffer.BufferedIndexValues();
218 bufferedOperations.put(keyBytes, values);
219 }
220
221 if(values.deletedIDs != null && values.deletedIDs.contains(entryID))
222 {
223 values.deletedIDs.remove(entryID);
224 return true;
225 }
226
227 if(values.addedIDs == null)
228 {
229 values.addedIDs = new EntryIDSet(keyBytes, null);
230 }
231
232 values.addedIDs.add(entryID);
233 return true;
234 }
235
236 /**
237 * Insert an entry ID into the set of IDs indexed by a given key.
238 *
239 * @param txn A database transaction, or null if none is required.
240 * @param key The index key.
241 * @param entryID The entry ID.
242 * @return True if the entry ID is inserted or ignored because the entry limit
243 * count is exceeded. False if it already exists in the entry ID set
244 * for the given key.
245 * @throws DatabaseException If an error occurs in the JE database.
246 */
247 public boolean insertID(Transaction txn, DatabaseEntry key, EntryID entryID)
248 throws DatabaseException
249 {
250 OperationStatus status;
251 DatabaseEntry entryIDData = entryID.getDatabaseEntry();
252 DatabaseEntry data = new DatabaseEntry();
253 boolean success = false;
254
255 if(maintainCount)
256 {
257 for(int i = 0; i < phantomWriteRetires; i++)
258 {
259 if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
260 OperationStatus.SUCCESS)
261 {
262 return true;
263 }
264 }
265 }
266 else
267 {
268 status = read(txn, key, data, LockMode.READ_COMMITTED);
269 if(status == OperationStatus.SUCCESS)
270 {
271 EntryIDSet entryIDList =
272 new EntryIDSet(key.getData(), data.getData());
273
274 if (entryIDList.isDefined())
275 {
276 for(int i = 0; i < phantomWriteRetires; i++)
277 {
278 if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
279 OperationStatus.SUCCESS)
280 {
281 return true;
282 }
283 }
284 }
285 }
286 else
287 {
288 if(rebuildRunning || trusted)
289 {
290 status = insert(txn, key, entryIDData);
291 if(status == OperationStatus.KEYEXIST)
292 {
293 for(int i = 1; i < phantomWriteRetires; i++)
294 {
295 if(insertIDWithRMW(txn, key, data, entryIDData, entryID) ==
296 OperationStatus.SUCCESS)
297 {
298 return true;
299 }
300 }
301 }
302 }
303 else
304 {
305 return true;
306 }
307 }
308 }
309
310 return success;
311 }
312
313
314 /**
315 * Add the specified import ID set to the provided key. Used during
316 * substring buffer flushing.
317 *
318 * @param txn A transaction.
319 * @param key The key to add the set to.
320 * @param importIdSet The set of import IDs.
321 * @param data Database entry to reuse for read
322 * @throws DatabaseException If an database error occurs.
323 */
324 public void insert(Transaction txn, DatabaseEntry key,
325 ImportIDSet importIdSet, DatabaseEntry data)
326 throws DatabaseException {
327
328 OperationStatus status;
329 status = read(txn, key, data, LockMode.RMW);
330 if(status == OperationStatus.SUCCESS) {
331 ImportIDSet newImportIDSet = new IntegerImportIDSet();
332 if (newImportIDSet.merge(data.getData(), importIdSet,
333 indexEntryLimit, maintainCount)) {
334 entryLimitExceededCount++;
335 }
336 data.setData(newImportIDSet.toDatabase());
337 } else if(status == OperationStatus.NOTFOUND) {
338 if(!importIdSet.isDefined()) {
339 entryLimitExceededCount++;
340 }
341 data.setData(importIdSet.toDatabase());
342 } else {
343 //Should never happen during import.
344 throw new DatabaseException();
345 }
346 put(txn,key, data);
347 }
348
349
350 /**
351 * Add the specified import ID set to the provided keys in the keyset.
352 *
353 * @param txn A transaction.
354 * @param importIDSet A import ID set to use.
355 * @param keySet The set containing the keys.
356 * @param keyData A key database entry to use.
357 * @param data A database entry to use for data.
358 * @return <CODE>True</CODE> if the insert was successful.
359 * @throws DatabaseException If a database error occurs.
360 */
361 public synchronized
362 boolean insert(Transaction txn, ImportIDSet importIDSet, Set<byte[]> keySet,
363 DatabaseEntry keyData, DatabaseEntry data)
364 throws DatabaseException {
365 for(byte[] key : keySet) {
366 keyData.setData(key);
367 insert(txn, keyData, importIDSet, data);
368 }
369 keyData.setData(null);
370 data.setData(null);
371 return true;
372 }
373
374 private OperationStatus insertIDWithRMW(Transaction txn, DatabaseEntry key,
375 DatabaseEntry data,
376 DatabaseEntry entryIDData,
377 EntryID entryID)
378 throws DatabaseException
379 {
380 OperationStatus status;
381
382 status = read(txn, key, data, LockMode.RMW);
383 if(status == OperationStatus.SUCCESS)
384 {
385 EntryIDSet entryIDList =
386 new EntryIDSet(key.getData(), data.getData());
387 if (entryIDList.isDefined() && indexEntryLimit > 0 &&
388 entryIDList.size() >= indexEntryLimit)
389 {
390 if(maintainCount)
391 {
392 entryIDList = new EntryIDSet(entryIDList.size());
393 }
394 else
395 {
396 entryIDList = new EntryIDSet();
397 }
398 entryLimitExceededCount++;
399
400 if(debugEnabled())
401 {
402 StringBuilder builder = new StringBuilder();
403 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
404 TRACER.debugInfo("Index entry exceeded in index %s. " +
405 "Limit: %d. ID list size: %d.\nKey:",
406 name, indexEntryLimit, entryIDList.size(),
407 builder);
408
409 }
410 }
411
412 entryIDList.add(entryID);
413
414 byte[] after = entryIDList.toDatabase();
415 data.setData(after);
416 return put(txn, key, data);
417 }
418 else
419 {
420 if(rebuildRunning || trusted)
421 {
422 return insert(txn, key, entryIDData);
423 }
424 else
425 {
426 return OperationStatus.SUCCESS;
427 }
428 }
429 }
430
431 /**
432 * Update the set of entry IDs for a given key.
433 *
434 * @param txn A database transaction, or null if none is required.
435 * @param key The database key.
436 * @param deletedIDs The IDs to remove for the key.
437 * @param addedIDs the IDs to add for the key.
438 * @throws DatabaseException If a database error occurs.
439 */
440 void updateKey(Transaction txn, DatabaseEntry key,
441 EntryIDSet deletedIDs, EntryIDSet addedIDs)
442 throws DatabaseException
443 {
444 OperationStatus status;
445 DatabaseEntry data = new DatabaseEntry();
446
447 // Handle cases where nothing is changed early to avoid
448 // DB access.
449 if(deletedIDs != null && deletedIDs.size() == 0 &&
450 (addedIDs == null || addedIDs.size() == 0))
451 {
452 return;
453 }
454
455 if(addedIDs != null && addedIDs.size() == 0 &&
456 (deletedIDs == null || deletedIDs.size() == 0))
457 {
458 return;
459 }
460
461
462 if(deletedIDs == null && addedIDs == null)
463 {
464 status = delete(txn, key);
465
466 if(status != OperationStatus.SUCCESS)
467 {
468 if(debugEnabled())
469 {
470 StringBuilder builder = new StringBuilder();
471 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
472 TRACER.debugError("The expected key does not exist in the " +
473 "index %s.\nKey:%s", name, builder.toString());
474 }
475 }
476
477 return;
478 }
479
480 if(maintainCount)
481 {
482 for(int i = 0; i < phantomWriteRetires; i++)
483 {
484 if(updateKeyWithRMW(txn, key, data, deletedIDs, addedIDs) ==
485 OperationStatus.SUCCESS)
486 {
487 return;
488 }
489 }
490 }
491 else
492 {
493 status = read(txn, key, data, LockMode.READ_COMMITTED);
494 if(status == OperationStatus.SUCCESS)
495 {
496 EntryIDSet entryIDList =
497 new EntryIDSet(key.getData(), data.getData());
498
499 if (entryIDList.isDefined())
500 {
501 for(int i = 0; i < phantomWriteRetires; i++)
502 {
503 if(updateKeyWithRMW(txn, key, data, deletedIDs, addedIDs) ==
504 OperationStatus.SUCCESS)
505 {
506 return;
507 }
508 }
509 }
510 }
511 else
512 {
513 if(rebuildRunning || trusted)
514 {
515 if(deletedIDs != null)
516 {
517 if(debugEnabled())
518 {
519 StringBuilder builder = new StringBuilder();
520 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
521 TRACER.debugError("The expected key does not exist in the " +
522 "index %s.\nKey:%s", name, builder.toString());
523 }
524 }
525 data.setData(addedIDs.toDatabase());
526
527 status = insert(txn, key, data);
528 if(status == OperationStatus.KEYEXIST)
529 {
530 for(int i = 1; i < phantomWriteRetires; i++)
531 {
532 if(updateKeyWithRMW(txn, key, data, deletedIDs, addedIDs) ==
533 OperationStatus.SUCCESS)
534 {
535 return;
536 }
537 }
538 }
539 }
540 }
541 }
542 }
543
544 private OperationStatus updateKeyWithRMW(Transaction txn,
545 DatabaseEntry key,
546 DatabaseEntry data,
547 EntryIDSet deletedIDs,
548 EntryIDSet addedIDs)
549 throws DatabaseException
550 {
551 OperationStatus status;
552
553 status = read(txn, key, data, LockMode.RMW);
554 if(status == OperationStatus.SUCCESS)
555 {
556 EntryIDSet entryIDList =
557 new EntryIDSet(key.getData(), data.getData());
558
559 if(addedIDs != null)
560 {
561 if(entryIDList.isDefined() && indexEntryLimit > 0)
562 {
563 long idCountDelta = addedIDs.size();
564 if(deletedIDs != null)
565 {
566 idCountDelta -= deletedIDs.size();
567 }
568 if(idCountDelta + entryIDList.size() >= indexEntryLimit)
569 {
570 if(maintainCount)
571 {
572 entryIDList = new EntryIDSet(entryIDList.size() + idCountDelta);
573 }
574 else
575 {
576 entryIDList = new EntryIDSet();
577 }
578 entryLimitExceededCount++;
579
580 if(debugEnabled())
581 {
582 StringBuilder builder = new StringBuilder();
583 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
584 TRACER.debugInfo("Index entry exceeded in index %s. " +
585 "Limit: %d. ID list size: %d.\nKey:",
586 name, indexEntryLimit, idCountDelta + addedIDs.size(),
587 builder);
588
589 }
590 }
591 else
592 {
593 entryIDList.addAll(addedIDs);
594 if(deletedIDs != null)
595 {
596 entryIDList.deleteAll(deletedIDs);
597 }
598 }
599 }
600 else
601 {
602 entryIDList.addAll(addedIDs);
603 if(deletedIDs != null)
604 {
605 entryIDList.deleteAll(deletedIDs);
606 }
607 }
608 }
609 else if(deletedIDs != null)
610 {
611 entryIDList.deleteAll(deletedIDs);
612 }
613
614 byte[] after = entryIDList.toDatabase();
615 if (after == null)
616 {
617 // No more IDs, so remove the key. If index is not
618 // trusted then this will cause all subsequent reads
619 // for this key to return undefined set.
620 return delete(txn, key);
621 }
622 else
623 {
624 data.setData(after);
625 return put(txn, key, data);
626 }
627 }
628 else
629 {
630 if(rebuildRunning || trusted)
631 {
632 if(deletedIDs != null)
633 {
634 if(debugEnabled())
635 {
636 StringBuilder builder = new StringBuilder();
637 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
638 TRACER.debugError("The expected key does not exist in the " +
639 "index %s.\nKey:%s", name, builder.toString());
640 }
641 }
642 data.setData(addedIDs.toDatabase());
643 return insert(txn, key, data);
644 }
645 else
646 {
647 return OperationStatus.SUCCESS;
648 }
649 }
650 }
651
652 /**
653 * Add an remove entry ID operation into a index buffer.
654 *
655 * @param buffer The index buffer to insert the ID into.
656 * @param keyBytes The index key bytes.
657 * @param entryID The entry ID.
658 * @return True if the entry ID is inserted or ignored because the entry limit
659 * count is exceeded. False if it already exists in the entry ID set
660 * for the given key.
661 */
662 public boolean removeID(IndexBuffer buffer, byte[] keyBytes,
663 EntryID entryID)
664 {
665 TreeMap<byte[], IndexBuffer.BufferedIndexValues> bufferedOperations =
666 buffer.getBufferedIndex(this);
667 IndexBuffer.BufferedIndexValues values = null;
668
669 if(bufferedOperations == null)
670 {
671 bufferedOperations = new TreeMap<byte[],
672 IndexBuffer.BufferedIndexValues>(comparator);
673 buffer.putBufferedIndex(this, bufferedOperations);
674 }
675 else
676 {
677 values = bufferedOperations.get(keyBytes);
678 }
679
680 if(values == null)
681 {
682 values = new IndexBuffer.BufferedIndexValues();
683 bufferedOperations.put(keyBytes, values);
684 }
685
686 if(values.addedIDs != null && values.addedIDs.contains(entryID))
687 {
688 values.addedIDs.remove(entryID);
689 return true;
690 }
691
692 if(values.deletedIDs == null)
693 {
694 values.deletedIDs = new EntryIDSet(keyBytes, null);
695 }
696
697 values.deletedIDs.add(entryID);
698 return true;
699 }
700
701 /**
702 * Remove an entry ID from the set of IDs indexed by a given key.
703 *
704 * @param txn A database transaction, or null if none is required.
705 * @param key The index key.
706 * @param entryID The entry ID.
707 * @throws DatabaseException If an error occurs in the JE database.
708 */
709 public void removeID(Transaction txn, DatabaseEntry key, EntryID entryID)
710 throws DatabaseException
711 {
712 OperationStatus status;
713 DatabaseEntry data = new DatabaseEntry();
714
715 if(maintainCount)
716 {
717 removeIDWithRMW(txn, key, data, entryID);
718 }
719 else
720 {
721 status = read(txn, key, data, LockMode.READ_COMMITTED);
722 if(status == OperationStatus.SUCCESS)
723 {
724 EntryIDSet entryIDList = new EntryIDSet(key.getData(), data.getData());
725 if(entryIDList.isDefined())
726 {
727 removeIDWithRMW(txn, key, data, entryID);
728 }
729 }
730 else
731 {
732 // Ignore failures if rebuild is running since a empty entryIDset
733 // will probably not be rebuilt.
734 if(trusted && !rebuildRunning)
735 {
736 setTrusted(txn, false);
737
738 if(debugEnabled())
739 {
740 StringBuilder builder = new StringBuilder();
741 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
742 TRACER.debugError("The expected key does not exist in the " +
743 "index %s.\nKey:%s", name, builder.toString());
744 }
745
746 logError(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD.get(name));
747 }
748 }
749 }
750 }
751
752 /**
753 * Delete specified entry ID from all keys in the provided key set.
754 *
755 * @param txn A Transaction.
756 * @param keySet A set of keys.
757 * @param entryID The entry ID to delete.
758 * @throws DatabaseException If a database error occurs.
759 */
760 public synchronized
761 void delete(Transaction txn, Set<byte[]> keySet, EntryID entryID)
762 throws DatabaseException {
763 setTrusted(txn, false);
764 for(byte[] key : keySet) {
765 removeIDWithRMW(txn, new DatabaseEntry(key),
766 new DatabaseEntry(), entryID);
767 }
768 setTrusted(txn, true);
769 }
770
771 private void removeIDWithRMW(Transaction txn, DatabaseEntry key,
772 DatabaseEntry data, EntryID entryID)
773 throws DatabaseException
774 {
775 OperationStatus status;
776 status = read(txn, key, data, LockMode.RMW);
777
778 if (status == OperationStatus.SUCCESS)
779 {
780 EntryIDSet entryIDList = new EntryIDSet(key.getData(), data.getData());
781 // Ignore failures if rebuild is running since the entry ID is
782 // probably already removed.
783 if (!entryIDList.remove(entryID) && !rebuildRunning)
784 {
785 if(trusted)
786 {
787 setTrusted(txn, false);
788
789 if(debugEnabled())
790 {
791 StringBuilder builder = new StringBuilder();
792 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
793 TRACER.debugError("The expected entry ID does not exist in " +
794 "the entry ID list for index %s.\nKey:%s",
795 name, builder.toString());
796 }
797
798 logError(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD.get(name));
799 }
800 }
801 else
802 {
803 byte[] after = entryIDList.toDatabase();
804 if (after == null)
805 {
806 // No more IDs, so remove the key. If index is not
807 // trusted then this will cause all subsequent reads
808 // for this key to return undefined set.
809 delete(txn, key);
810 }
811 else
812 {
813 data.setData(after);
814 put(txn, key, data);
815 }
816 }
817 }
818 else
819 {
820 // Ignore failures if rebuild is running since a empty entryIDset
821 // will probably not be rebuilt.
822 if(trusted && !rebuildRunning)
823 {
824 setTrusted(txn, false);
825
826 if(debugEnabled())
827 {
828 StringBuilder builder = new StringBuilder();
829 StaticUtils.byteArrayToHexPlusAscii(builder, key.getData(), 4);
830 TRACER.debugError("The expected key does not exist in the " +
831 "index %s.\nKey:%s", name, builder.toString());
832 }
833
834 logError(ERR_JEB_INDEX_CORRUPT_REQUIRES_REBUILD.get(name));
835 }
836 }
837 }
838
839 /**
840 * Buffered delete of a key from the JE database.
841 * @param buffer The index buffer to use to store the deleted keys
842 * @param keyBytes The index key bytes.
843 */
844 public void delete(IndexBuffer buffer, byte[] keyBytes)
845 {
846 TreeMap<byte[], IndexBuffer.BufferedIndexValues> bufferedOperations =
847 buffer.getBufferedIndex(this);
848 IndexBuffer.BufferedIndexValues values = null;
849
850 if(bufferedOperations == null)
851 {
852 bufferedOperations = new TreeMap<byte[],
853 IndexBuffer.BufferedIndexValues>(comparator);
854 buffer.putBufferedIndex(this, bufferedOperations);
855 }
856 else
857 {
858 values = bufferedOperations.get(keyBytes);
859 }
860
861 if(values == null)
862 {
863 values = new IndexBuffer.BufferedIndexValues();
864 bufferedOperations.put(keyBytes, values);
865 }
866 }
867
868 /**
869 * Check if an entry ID is in the set of IDs indexed by a given key.
870 *
871 * @param txn A database transaction, or null if none is required.
872 * @param key The index key.
873 * @param entryID The entry ID.
874 * @return true if the entry ID is indexed by the given key,
875 * false if it is not indexed by the given key,
876 * undefined if the key has exceeded the entry limit.
877 * @throws DatabaseException If an error occurs in the JE database.
878 */
879 public ConditionResult containsID(Transaction txn, DatabaseEntry key,
880 EntryID entryID)
881 throws DatabaseException
882 {
883 if(rebuildRunning)
884 {
885 return ConditionResult.UNDEFINED;
886 }
887
888 OperationStatus status;
889 LockMode lockMode = LockMode.DEFAULT;
890 DatabaseEntry data = new DatabaseEntry();
891
892 status = read(txn, key, data, lockMode);
893 if (status == OperationStatus.SUCCESS)
894 {
895 EntryIDSet entryIDList =
896 new EntryIDSet(key.getData(), data.getData());
897
898 if (!entryIDList.isDefined())
899 {
900 return ConditionResult.UNDEFINED;
901 }
902 else if (entryIDList.contains(entryID))
903 {
904 return ConditionResult.TRUE;
905 }
906 else
907 {
908 return ConditionResult.FALSE;
909 }
910 }
911 else
912 {
913 if(trusted)
914 {
915 return ConditionResult.FALSE;
916 }
917 else
918 {
919 return ConditionResult.UNDEFINED;
920 }
921 }
922 }
923
924 /**
925 * Reads the set of entry IDs for a given key.
926 *
927 * @param key The database key.
928 * @param txn A database transaction, or null if none is required.
929 * @param lockMode The JE locking mode to be used for the database read.
930 * @return The entry IDs indexed by this key.
931 */
932 public EntryIDSet readKey(DatabaseEntry key, Transaction txn,
933 LockMode lockMode)
934 {
935 if(rebuildRunning)
936 {
937 return new EntryIDSet();
938 }
939
940 try
941 {
942 OperationStatus status;
943 DatabaseEntry data = new DatabaseEntry();
944 status = read( txn, key, data, lockMode);
945 if (status != OperationStatus.SUCCESS)
946 {
947 if(trusted)
948 {
949 return new EntryIDSet(key.getData(), null);
950 }
951 else
952 {
953 return new EntryIDSet();
954 }
955 }
956 return new EntryIDSet(key.getData(), data.getData());
957 }
958 catch (DatabaseException e)
959 {
960 if (debugEnabled())
961 {
962 TRACER.debugCaught(DebugLogLevel.ERROR, e);
963 }
964 return new EntryIDSet();
965 }
966 }
967
968 /**
969 * Writes the set of entry IDs for a given key.
970 *
971 * @param key The database key.
972 * @param entryIDList The entry IDs indexed by this key.
973 * @param txn A database transaction, or null if none is required.
974 * @throws DatabaseException If an error occurs in the JE database.
975 */
976 public void writeKey(Transaction txn, DatabaseEntry key,
977 EntryIDSet entryIDList)
978 throws DatabaseException
979 {
980 DatabaseEntry data = new DatabaseEntry();
981 byte[] after = entryIDList.toDatabase();
982 if (after == null)
983 {
984 // No more IDs, so remove the key.
985 delete(txn, key);
986 }
987 else
988 {
989 if (!entryIDList.isDefined())
990 {
991 entryLimitExceededCount++;
992 }
993 data.setData(after);
994 put(txn, key, data);
995 }
996 }
997
998 /**
999 * Reads a range of keys and collects all their entry IDs into a
1000 * single set.
1001 *
1002 * @param lower The lower bound of the range. A 0 length byte array indicates
1003 * no lower bound and the range will start from the
1004 * smallest key.
1005 * @param upper The upper bound of the range. A 0 length byte array indicates
1006 * no upper bound and the range will end at the largest
1007 * key.
1008 * @param lowerIncluded true if a key exactly matching the lower bound
1009 * is included in the range, false if only keys
1010 * strictly greater than the lower bound are included.
1011 * This value is ignored if the lower bound is not
1012 * specified.
1013 * @param upperIncluded true if a key exactly matching the upper bound
1014 * is included in the range, false if only keys
1015 * strictly less than the upper bound are included.
1016 * This value is ignored if the upper bound is not
1017 * specified.
1018 * @return The set of entry IDs.
1019 */
1020 public EntryIDSet readRange(byte[] lower, byte[] upper,
1021 boolean lowerIncluded, boolean upperIncluded)
1022 {
1023 LockMode lockMode = LockMode.DEFAULT;
1024
1025 // If this index is not trusted, then just return an undefined
1026 // id set.
1027 if(rebuildRunning || !trusted)
1028 {
1029 return new EntryIDSet();
1030 }
1031
1032 try
1033 {
1034 // Total number of IDs found so far.
1035 int totalIDCount = 0;
1036
1037 DatabaseEntry data = new DatabaseEntry();
1038 DatabaseEntry key;
1039
1040 ArrayList<EntryIDSet> lists = new ArrayList<EntryIDSet>();
1041
1042 OperationStatus status;
1043 Cursor cursor;
1044
1045 cursor = openCursor(null, CursorConfig.READ_COMMITTED);
1046
1047 try
1048 {
1049 // Set the lower bound if necessary.
1050 if(lower.length > 0)
1051 {
1052 key = new DatabaseEntry(lower);
1053
1054 // Initialize the cursor to the lower bound.
1055 status = cursor.getSearchKeyRange(key, data, lockMode);
1056
1057 // Advance past the lower bound if necessary.
1058 if (status == OperationStatus.SUCCESS && !lowerIncluded &&
1059 comparator.compare(key.getData(), lower) == 0)
1060 {
1061 // Do not include the lower value.
1062 status = cursor.getNext(key, data, lockMode);
1063 }
1064 }
1065 else
1066 {
1067 key = new DatabaseEntry();
1068 status = cursor.getNext(key, data, lockMode);
1069 }
1070
1071 if (status != OperationStatus.SUCCESS)
1072 {
1073 // There are no values.
1074 return new EntryIDSet(key.getData(), null);
1075 }
1076
1077 // Step through the keys until we hit the upper bound or the last key.
1078 while (status == OperationStatus.SUCCESS)
1079 {
1080 // Check against the upper bound if necessary
1081 if(upper.length > 0)
1082 {
1083 int cmp = comparator.compare(key.getData(), upper);
1084 if ((cmp > 0) || (cmp == 0 && !upperIncluded))
1085 {
1086 break;
1087 }
1088 }
1089 EntryIDSet list = new EntryIDSet(key.getData(), data.getData());
1090 if (!list.isDefined())
1091 {
1092 // There is no point continuing.
1093 return list;
1094 }
1095 totalIDCount += list.size();
1096 if (cursorEntryLimit > 0 && totalIDCount > cursorEntryLimit)
1097 {
1098 // There are too many. Give up and return an undefined list.
1099 return new EntryIDSet();
1100 }
1101 lists.add(list);
1102 status = cursor.getNext(key, data, LockMode.DEFAULT);
1103 }
1104
1105 return EntryIDSet.unionOfSets(lists, false);
1106 }
1107 finally
1108 {
1109 cursor.close();
1110 }
1111 }
1112 catch (DatabaseException e)
1113 {
1114 if (debugEnabled())
1115 {
1116 TRACER.debugCaught(DebugLogLevel.ERROR, e);
1117 }
1118 return new EntryIDSet();
1119 }
1120 }
1121
1122 /**
1123 * Get the number of keys that have exceeded the entry limit since this
1124 * object was created.
1125 * @return The number of keys that have exceeded the entry limit since this
1126 * object was created.
1127 */
1128 public int getEntryLimitExceededCount()
1129 {
1130 return entryLimitExceededCount;
1131 }
1132
1133 /**
1134 * Increment the count of the number of keys that have exceeded the entry
1135 * limit since this object was created.
1136 */
1137 public void incEntryLimitExceededCount()
1138 {
1139 entryLimitExceededCount++;
1140 }
1141
1142 /**
1143 * Update the index buffer for a deleted entry.
1144 *
1145 * @param buffer The index buffer to use to store the deleted keys
1146 * @param entryID The entry ID.
1147 * @param entry The entry to be indexed.
1148 * @return True if all the indexType keys for the entry are added. False if
1149 * the entry ID already exists for some keys.
1150 * @throws DatabaseException If an error occurs in the JE database.
1151 * @throws DirectoryException If a Directory Server error occurs.
1152 */
1153 public boolean addEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
1154 throws DatabaseException, DirectoryException
1155 {
1156 HashSet<byte[]> addKeys = new HashSet<byte[]>();
1157 boolean success = true;
1158
1159 indexer.indexEntry(entry, addKeys);
1160
1161 for (byte[] keyBytes : addKeys)
1162 {
1163 if(!insertID(buffer, keyBytes, entryID))
1164 {
1165 success = false;
1166 }
1167 }
1168
1169 return success;
1170 }
1171
1172 /**
1173 * Update the index for a new entry.
1174 *
1175 * @param txn A database transaction, or null if none is required.
1176 * @param entryID The entry ID.
1177 * @param entry The entry to be indexed.
1178 * @return True if all the indexType keys for the entry are added. False if
1179 * the entry ID already exists for some keys.
1180 * @throws DatabaseException If an error occurs in the JE database.
1181 * @throws DirectoryException If a Directory Server error occurs.
1182 */
1183 public boolean addEntry(Transaction txn, EntryID entryID, Entry entry)
1184 throws DatabaseException, DirectoryException
1185 {
1186 TreeSet<byte[]> addKeys = new TreeSet<byte[]>(indexer.getComparator());
1187 boolean success = true;
1188
1189 indexer.indexEntry(entry, addKeys);
1190
1191 DatabaseEntry key = new DatabaseEntry();
1192 for (byte[] keyBytes : addKeys)
1193 {
1194 key.setData(keyBytes);
1195 if(!insertID(txn, key, entryID))
1196 {
1197 success = false;
1198 }
1199 }
1200
1201 return success;
1202 }
1203
1204 /**
1205 * Update the index buffer for a deleted entry.
1206 *
1207 * @param buffer The index buffer to use to store the deleted keys
1208 * @param entryID The entry ID
1209 * @param entry The contents of the deleted entry.
1210 * @throws DatabaseException If an error occurs in the JE database.
1211 * @throws DirectoryException If a Directory Server error occurs.
1212 */
1213 public void removeEntry(IndexBuffer buffer, EntryID entryID, Entry entry)
1214 throws DatabaseException, DirectoryException
1215 {
1216 HashSet<byte[]> delKeys = new HashSet<byte[]>();
1217
1218 indexer.indexEntry(entry, delKeys);
1219
1220 for (byte[] keyBytes : delKeys)
1221 {
1222 removeID(buffer, keyBytes, entryID);
1223 }
1224 }
1225
1226 /**
1227 * Update the index for a deleted entry.
1228 *
1229 * @param txn A database transaction, or null if none is required.
1230 * @param entryID The entry ID
1231 * @param entry The contents of the deleted entry.
1232 * @throws DatabaseException If an error occurs in the JE database.
1233 * @throws DirectoryException If a Directory Server error occurs.
1234 */
1235 public void removeEntry(Transaction txn, EntryID entryID, Entry entry)
1236 throws DatabaseException, DirectoryException
1237 {
1238 TreeSet<byte[]> delKeys = new TreeSet<byte[]>(indexer.getComparator());
1239
1240 indexer.indexEntry(entry, delKeys);
1241
1242 DatabaseEntry key = new DatabaseEntry();
1243 for (byte[] keyBytes : delKeys)
1244 {
1245 key.setData(keyBytes);
1246 removeID(txn, key, entryID);
1247 }
1248 }
1249
1250
1251 /**
1252 * Update the index to reflect a sequence of modifications in a Modify
1253 * operation.
1254 *
1255 * @param txn A database transaction, or null if none is required.
1256 * @param entryID The ID of the entry that was modified.
1257 * @param oldEntry The entry before the modifications were applied.
1258 * @param newEntry The entry after the modifications were applied.
1259 * @param mods The sequence of modifications in the Modify operation.
1260 * @throws DatabaseException If an error occurs in the JE database.
1261 */
1262 public void modifyEntry(Transaction txn,
1263 EntryID entryID,
1264 Entry oldEntry,
1265 Entry newEntry,
1266 List<Modification> mods)
1267 throws DatabaseException
1268 {
1269 TreeMap<byte[], Boolean> modifiedKeys =
1270 new TreeMap<byte[], Boolean>(indexer.getComparator());
1271
1272 indexer.modifyEntry(oldEntry, newEntry, mods, modifiedKeys);
1273
1274 DatabaseEntry key = new DatabaseEntry();
1275 for (Map.Entry<byte[], Boolean> modifiedKey : modifiedKeys.entrySet())
1276 {
1277 key.setData(modifiedKey.getKey());
1278 if(modifiedKey.getValue())
1279 {
1280 insertID(txn, key, entryID);
1281 }
1282 else
1283 {
1284 removeID(txn, key, entryID);
1285 }
1286 }
1287 }
1288
1289 /**
1290 * Update the index to reflect a sequence of modifications in a Modify
1291 * operation.
1292 *
1293 * @param buffer The index buffer to use to store the deleted keys
1294 * @param entryID The ID of the entry that was modified.
1295 * @param oldEntry The entry before the modifications were applied.
1296 * @param newEntry The entry after the modifications were applied.
1297 * @param mods The sequence of modifications in the Modify operation.
1298 * @throws DatabaseException If an error occurs in the JE database.
1299 */
1300 public void modifyEntry(IndexBuffer buffer,
1301 EntryID entryID,
1302 Entry oldEntry,
1303 Entry newEntry,
1304 List<Modification> mods)
1305 throws DatabaseException
1306 {
1307 HashMap<byte[], Boolean> modifiedKeys = new HashMap<byte[], Boolean>();
1308
1309 indexer.modifyEntry(oldEntry, newEntry, mods, modifiedKeys);
1310 for (Map.Entry<byte[], Boolean> modifiedKey : modifiedKeys.entrySet())
1311 {
1312 if(modifiedKey.getValue())
1313 {
1314 insertID(buffer, modifiedKey.getKey(), entryID);
1315 }
1316 else
1317 {
1318 removeID(buffer, modifiedKey.getKey(), entryID);
1319 }
1320 }
1321 }
1322
1323 /**
1324 * Set the index entry limit.
1325 *
1326 * @param indexEntryLimit The index entry limit to set.
1327 * @return True if a rebuild is required or false otherwise.
1328 */
1329 public boolean setIndexEntryLimit(int indexEntryLimit)
1330 {
1331 boolean rebuildRequired = false;
1332 if(this.indexEntryLimit < indexEntryLimit &&
1333 entryLimitExceededCount > 0 )
1334 {
1335 rebuildRequired = true;
1336 }
1337 this.indexEntryLimit = indexEntryLimit;
1338
1339 return rebuildRequired;
1340 }
1341
1342 /**
1343 * Set the indexer.
1344 *
1345 * @param indexer The indexer to set
1346 */
1347 public void setIndexer(Indexer indexer)
1348 {
1349 this.indexer = indexer;
1350 }
1351
1352 /**
1353 * Return entry limit.
1354 *
1355 * @return The entry limit.
1356 */
1357 public int getIndexEntryLimit() {
1358 return this.indexEntryLimit;
1359 }
1360
1361 /**
1362 * Set the index trust state.
1363 * @param txn A database transaction, or null if none is required.
1364 * @param trusted True if this index should be trusted or false
1365 * otherwise.
1366 * @throws DatabaseException If an error occurs in the JE database.
1367 */
1368 public synchronized void setTrusted(Transaction txn, boolean trusted)
1369 throws DatabaseException
1370 {
1371 this.trusted = trusted;
1372 state.putIndexTrustState(txn, this, trusted);
1373 }
1374
1375 /**
1376 * Return true iff this index is trusted.
1377 * @return the trusted state of this index
1378 */
1379 public synchronized boolean isTrusted()
1380 {
1381 return trusted;
1382 }
1383
1384 /**
1385 * Set the rebuild status of this index.
1386 * @param rebuildRunning True if a rebuild process on this index
1387 * is running or False otherwise.
1388 */
1389 public synchronized void setRebuildStatus(boolean rebuildRunning)
1390 {
1391 this.rebuildRunning = rebuildRunning;
1392 }
1393
1394 /**
1395 * Whether this index maintains a count of IDs for keys once the
1396 * entry limit has exceeded.
1397 * @return <code>true</code> if this index maintains court of IDs
1398 * or <code>false</code> otherwise
1399 */
1400 public boolean getMaintainCount()
1401 {
1402 return maintainCount;
1403 }
1404 }