Package org.apache.lucene.index

Class DocumentsWriterFlushControl

java.lang.Object

org.apache.lucene.index.DocumentsWriterFlushControl

All Implemented Interfaces:

java.io.Closeable, java.lang.AutoCloseable, Accountable

final class DocumentsWriterFlushControl
extends java.lang.Object
implements Accountable, java.io.Closeable

This class controls DocumentsWriterPerThread flushing during indexing. It tracks the memory consumption per DocumentsWriterPerThread and uses a configured FlushPolicy to decide if a DocumentsWriterPerThread must flush.

In addition to the FlushPolicy the flush control might set certain DocumentsWriterPerThread as flush pending iff a DocumentsWriterPerThread exceeds the IndexWriterConfig.getRAMPerThreadHardLimitMB() to prevent address space exhaustion.

Field Summary

Fields

Modifier and Type	Field	Description
private long	activeBytes
private java.util.Queue<DocumentsWriterPerThread>	blockedFlushes
private boolean	closed
private LiveIndexWriterConfig	config
private DocumentsWriter	documentsWriter
private boolean	flushByRAMWasDisabled
private long	flushBytes
private java.util.concurrent.atomic.AtomicBoolean	flushDeletes
private java.util.List<DocumentsWriterPerThread>	flushingWriters
private FlushPolicy	flushPolicy
private java.util.Queue<DocumentsWriterPerThread>	flushQueue
private boolean	fullFlush
private boolean	fullFlushMarkDone
private long	hardMaxBytesPerDWPT
private InfoStream	infoStream
private double	maxConfiguredRamBuffer
private int	numDocsSinceStalled
private int	numPending
private long	peakActiveBytes
private long	peakDelta
private long	peakFlushBytes
private long	peakNetBytes
private DocumentsWriterPerThreadPool	perThreadPool
(package private) DocumentsWriterStallControl	stallControl
private long	stallStartNS

Fields inherited from interface org.apache.lucene.util.Accountable

NULL_ACCOUNTABLE

Constructor Summary

Constructors

Constructor	Description
DocumentsWriterFlushControl(DocumentsWriter documentsWriter, LiveIndexWriterConfig config)

Method Summary

Modifier and Type	Method	Description
(package private) void	abortFullFlushes()
(package private) void	abortPendingFlushes()
long	activeBytes()
private void	addFlushingDWPT(DocumentsWriterPerThread perThread)
java.util.Iterator<DocumentsWriterPerThread>	allActiveWriters()	Returns an iterator that provides access to all currently active DocumentsWriterPerThreads
(package private) boolean	anyStalledThreads()	Returns true iff stalled
private boolean	assertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)
(package private) boolean	assertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)
private boolean	assertMemory()
private boolean	assertNumDocsSinceStalled(boolean stalled)
private DocumentsWriterPerThread	checkout(DocumentsWriterPerThread perThread, boolean markPending)
private void	checkoutAndBlock(DocumentsWriterPerThread perThread)
private DocumentsWriterPerThread	checkOutForFlush(DocumentsWriterPerThread perThread)
(package private) DocumentsWriterPerThread	checkoutLargestNonPendingWriter()	Returns the largest non-pending flushable DWPT or null if there is none.
void	close()
private void	commitPerThreadBytes(DocumentsWriterPerThread perThread)
(package private) DocumentsWriterPerThread	doAfterDocument(DocumentsWriterPerThread perThread, boolean isUpdate)
(package private) void	doAfterFlush(DocumentsWriterPerThread dwpt)
(package private) void	doOnAbort(DocumentsWriterPerThread perThread)
(package private) void	doOnDelete()
(package private) DocumentsWriterPerThread	findLargestNonPendingWriter()
(package private) void	finishFullFlush()
boolean	getAndResetApplyAllDeletes()
long	getDeleteBytesUsed()	Returns heap bytes currently consumed by buffered deletes/updates that would be freed if we pushed all deletes.
(package private) long	getFlushingBytes()
InfoStream	getInfoStream()	Returns the IndexWriter InfoStream
(package private) long	getPeakActiveBytes()
(package private) long	getPeakNetBytes()
(package private) boolean	isFullFlush()	Returns true if a full flush is currently running
(package private) long	markForFullFlush()
(package private) long	netBytes()
(package private) DocumentsWriterPerThread	nextPendingFlush()
(package private) int	numBlockedFlushes()	Returns the number of flushes that are checked out but not yet available for flushing.
(package private) int	numFlushingDWPT()
(package private) int	numQueuedFlushes()	Returns the number of flushes that are already checked out but not yet actively flushing
(package private) DocumentsWriterPerThread	obtainAndLock()
private void	pruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)	Prunes the blockedQueue by removing all DWPTs that are associated with the given flush queue.
long	ramBytesUsed()	Return the memory usage of this object in bytes.
void	setApplyAllDeletes()
void	setFlushPending(DocumentsWriterPerThread perThread)	Sets flush pending state on the given DocumentsWriterPerThread.
private long	stallLimitBytes()
java.lang.String	toString()
private boolean	updatePeaks(long delta)
private boolean	updateStallState()
void	waitForFlush()
(package private) void	waitIfStalled()	This method will block if too many DWPT are currently flushing and no checked out DWPT are available

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, wait, wait, wait

Methods inherited from interface org.apache.lucene.util.Accountable

getChildResources

Field Detail

hardMaxBytesPerDWPT

private final long hardMaxBytesPerDWPT

activeBytes

private long activeBytes

flushBytes

private volatile long flushBytes

numPending

private volatile int numPending

numDocsSinceStalled

private int numDocsSinceStalled

flushDeletes

private final java.util.concurrent.atomic.AtomicBoolean flushDeletes

fullFlush

private boolean fullFlush

fullFlushMarkDone

private boolean fullFlushMarkDone

flushQueue

private final java.util.Queue<DocumentsWriterPerThread> flushQueue

blockedFlushes

private final java.util.Queue<DocumentsWriterPerThread> blockedFlushes

flushingWriters

private final java.util.List<DocumentsWriterPerThread> flushingWriters

maxConfiguredRamBuffer

private double maxConfiguredRamBuffer

peakActiveBytes

private long peakActiveBytes

peakFlushBytes

private long peakFlushBytes

peakNetBytes

private long peakNetBytes

peakDelta

private long peakDelta

flushByRAMWasDisabled

private boolean flushByRAMWasDisabled

stallControl

final DocumentsWriterStallControl stallControl

perThreadPool

private final DocumentsWriterPerThreadPool perThreadPool

flushPolicy

private final FlushPolicy flushPolicy

closed

private boolean closed

documentsWriter

private final DocumentsWriter documentsWriter

config

private final LiveIndexWriterConfig config

infoStream

private final InfoStream infoStream

stallStartNS

private long stallStartNS

Constructor Detail

DocumentsWriterFlushControl

DocumentsWriterFlushControl(DocumentsWriter documentsWriter,
                            LiveIndexWriterConfig config)

Method Detail

activeBytes

public long activeBytes()

getFlushingBytes

long getFlushingBytes()

netBytes

long netBytes()

stallLimitBytes

private long stallLimitBytes()

assertMemory

private boolean assertMemory()

commitPerThreadBytes

private void commitPerThreadBytes(DocumentsWriterPerThread perThread)

updatePeaks

private boolean updatePeaks(long delta)

doAfterDocument

DocumentsWriterPerThread doAfterDocument(DocumentsWriterPerThread perThread,
                                         boolean isUpdate)

checkout

private DocumentsWriterPerThread checkout(DocumentsWriterPerThread perThread,
                                          boolean markPending)

assertNumDocsSinceStalled

private boolean assertNumDocsSinceStalled(boolean stalled)

doAfterFlush

void doAfterFlush(DocumentsWriterPerThread dwpt)

updateStallState

private boolean updateStallState()

waitForFlush

public void waitForFlush()

setFlushPending

public void setFlushPending(DocumentsWriterPerThread perThread)

Sets flush pending state on the given DocumentsWriterPerThread. The DocumentsWriterPerThread must have indexed at least on Document and must not be already pending.

doOnAbort

void doOnAbort(DocumentsWriterPerThread perThread)

checkoutAndBlock

private void checkoutAndBlock(DocumentsWriterPerThread perThread)

checkOutForFlush

private DocumentsWriterPerThread checkOutForFlush(DocumentsWriterPerThread perThread)

addFlushingDWPT

private void addFlushingDWPT(DocumentsWriterPerThread perThread)

toString

public java.lang.String toString()

Overrides:

toString in class java.lang.Object

nextPendingFlush

DocumentsWriterPerThread nextPendingFlush()

close

public void close()

Specified by:

close in interface java.lang.AutoCloseable

Specified by:

close in interface java.io.Closeable

allActiveWriters

public java.util.Iterator<DocumentsWriterPerThread> allActiveWriters()

Returns an iterator that provides access to all currently active DocumentsWriterPerThreads

doOnDelete

void doOnDelete()

getDeleteBytesUsed

public long getDeleteBytesUsed()

Returns heap bytes currently consumed by buffered deletes/updates that would be freed if we pushed all deletes. This does not include bytes consumed by already pushed delete/update packets.

ramBytesUsed

public long ramBytesUsed()

Description copied from interface: Accountable

Return the memory usage of this object in bytes. Negative values are illegal.

Specified by:

ramBytesUsed in interface Accountable

numFlushingDWPT

int numFlushingDWPT()

getAndResetApplyAllDeletes

public boolean getAndResetApplyAllDeletes()

setApplyAllDeletes

public void setApplyAllDeletes()

obtainAndLock

DocumentsWriterPerThread obtainAndLock()

markForFullFlush

long markForFullFlush()

assertActiveDeleteQueue

private boolean assertActiveDeleteQueue(DocumentsWriterDeleteQueue queue)

pruneBlockedQueue

private void pruneBlockedQueue(DocumentsWriterDeleteQueue flushingQueue)

Prunes the blockedQueue by removing all DWPTs that are associated with the given flush queue.

finishFullFlush

void finishFullFlush()

assertBlockedFlushes

boolean assertBlockedFlushes(DocumentsWriterDeleteQueue flushingQueue)

abortFullFlushes

void abortFullFlushes()

abortPendingFlushes

void abortPendingFlushes()

isFullFlush

boolean isFullFlush()

Returns true if a full flush is currently running

numQueuedFlushes

int numQueuedFlushes()

Returns the number of flushes that are already checked out but not yet actively flushing

numBlockedFlushes

int numBlockedFlushes()

Returns the number of flushes that are checked out but not yet available for flushing. This only applies during a full flush if a DWPT needs flushing but must not be flushed until the full flush has finished.

waitIfStalled

void waitIfStalled()

This method will block if too many DWPT are currently flushing and no checked out DWPT are available

anyStalledThreads

boolean anyStalledThreads()

Returns true iff stalled

getInfoStream

public InfoStream getInfoStream()

Returns the IndexWriter InfoStream

findLargestNonPendingWriter

DocumentsWriterPerThread findLargestNonPendingWriter()

checkoutLargestNonPendingWriter

final DocumentsWriterPerThread checkoutLargestNonPendingWriter()

Returns the largest non-pending flushable DWPT or null if there is none.

getPeakActiveBytes

long getPeakActiveBytes()

getPeakNetBytes

long getPeakNetBytes()