You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/08/01 20:42:12 UTC
svn commit: r1509380 - in
/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index:
BufferedDeletes.java DocumentsWriterPerThread.java
FlushByRamOrCountsPolicy.java FlushPolicy.java LiveIndexWriterConfig.java
SegmentWriteState.java
Author: shaie
Date: Thu Aug 1 18:42:11 2013
New Revision: 1509380
URL: http://svn.apache.org/r1509380
Log:
some javadocs and comments improvements
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1509380&r1=1509379&r2=1509380&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BufferedDeletes.java Thu Aug 1 18:42:11 2013
@@ -33,9 +33,9 @@ import org.apache.lucene.util.RamUsageEs
* deletes are pushed (on flush in DocumentsWriter), these
* deletes are converted to a FrozenDeletes instance. */
-// NOTE: we are sync'd by BufferedDeletes, ie, all access to
-// instances of this class is via sync'd methods on
-// BufferedDeletes
+// NOTE: instances of this class are accessed either via a private
+// instance on DocumentWriterPerThread, or via sync'd code by
+// DocumentsWriterDeleteQueue
class BufferedDeletes {
@@ -136,6 +136,9 @@ class BufferedDeletes {
}
terms.put(term, Integer.valueOf(docIDUpto));
+ // note that if current != null then it means there's already a buffered
+ // delete on that term, therefore we seem to over-count. this over-counting
+ // is done to respect IndexWriterConfig.setMaxBufferedDeleteTerms.
numTermDeletes.incrementAndGet();
if (current == null) {
bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.bytes.length + (RamUsageEstimator.NUM_BYTES_CHAR * term.field().length()));
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1509380&r1=1509379&r2=1509380&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Thu Aug 1 18:42:11 2013
@@ -403,8 +403,8 @@ class DocumentsWriterPerThread {
++numDocsInRAM;
}
- // Buffer a specific docID for deletion. Currently only
- // used when we hit a exception when adding a document
+ // Buffer a specific docID for deletion. Currently only
+ // used when we hit an exception when adding a document
void deleteDocID(int docIDUpto) {
pendingDeletes.addDocID(docIDUpto);
// NOTE: we do not trigger flush here. This is
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java?rev=1509380&r1=1509379&r2=1509380&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java Thu Aug 1 18:42:11 2013
@@ -20,23 +20,32 @@ package org.apache.lucene.index;
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
/**
- * Default {@link FlushPolicy} implementation that flushes based on RAM used,
- * document count and number of buffered deletes depending on the IndexWriter's
- * {@link IndexWriterConfig}.
+ * Default {@link FlushPolicy} implementation that flushes new segments based on
+ * RAM used and document count depending on the IndexWriter's
+ * {@link IndexWriterConfig}. It also applies pending deletes based on the
+ * number of buffered delete terms.
*
* <ul>
- * <li>{@link #onDelete(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)} - flushes
- * based on the global number of buffered delete terms iff
- * {@link IndexWriterConfig#getMaxBufferedDeleteTerms()} is enabled</li>
- * <li>{@link #onInsert(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)} - flushes
- * either on the number of documents per {@link DocumentsWriterPerThread} (
+ * <li>
+ * {@link #onDelete(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)}
+ * - applies pending delete operations based on the global number of buffered
+ * delete terms iff {@link IndexWriterConfig#getMaxBufferedDeleteTerms()} is
+ * enabled</li>
+ * <li>
+ * {@link #onInsert(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)}
+ * - flushes either on the number of documents per
+ * {@link DocumentsWriterPerThread} (
* {@link DocumentsWriterPerThread#getNumDocsInRAM()}) or on the global active
* memory consumption in the current indexing session iff
* {@link IndexWriterConfig#getMaxBufferedDocs()} or
* {@link IndexWriterConfig#getRAMBufferSizeMB()} is enabled respectively</li>
- * <li>{@link #onUpdate(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)} - calls
- * {@link #onInsert(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)} and
- * {@link #onDelete(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)} in order</li>
+ * <li>
+ * {@link #onUpdate(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)}
+ * - calls
+ * {@link #onInsert(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)}
+ * and
+ * {@link #onDelete(DocumentsWriterFlushControl, DocumentsWriterPerThreadPool.ThreadState)}
+ * in order</li>
* </ul>
* All {@link IndexWriterConfig} settings are used to mark
* {@link DocumentsWriterPerThread} as flush pending during indexing with
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java?rev=1509380&r1=1509379&r2=1509380&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FlushPolicy.java Thu Aug 1 18:42:11 2013
@@ -32,18 +32,19 @@ import org.apache.lucene.util.SetOnce;
* {@link IndexWriterConfig#setRAMBufferSizeMB(double)}</li>
* <li>Number of RAM resident documents - configured via
* {@link IndexWriterConfig#setMaxBufferedDocs(int)}</li>
- * <li>Number of buffered delete terms/queries - configured via
- * {@link IndexWriterConfig#setMaxBufferedDeleteTerms(int)}</li>
* </ul>
- *
- * The {@link IndexWriter} consults a provided {@link FlushPolicy} to control the
- * flushing process. The policy is informed for each added or
- * updated document as well as for each delete term. Based on the
- * {@link FlushPolicy}, the information provided via {@link ThreadState} and
+ * The policy also applies pending delete operations (by term and/or query),
+ * given the threshold set in
+ * {@link IndexWriterConfig#setMaxBufferedDeleteTerms(int)}.
+ * <p>
+ * {@link IndexWriter} consults the provided {@link FlushPolicy} to control the
+ * flushing process. The policy is informed for each added or updated document
+ * as well as for each delete term. Based on the {@link FlushPolicy}, the
+ * information provided via {@link ThreadState} and
* {@link DocumentsWriterFlushControl}, the {@link FlushPolicy} decides if a
- * {@link DocumentsWriterPerThread} needs flushing and mark it as
- * flush-pending via
- * {@link DocumentsWriterFlushControl#setFlushPending(DocumentsWriterPerThreadPool.ThreadState)}.
+ * {@link DocumentsWriterPerThread} needs flushing and mark it as flush-pending
+ * via {@link DocumentsWriterFlushControl#setFlushPending}, or if deletes need
+ * to be applied.
*
* @see ThreadState
* @see DocumentsWriterFlushControl
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java?rev=1509380&r1=1509379&r2=1509380&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/LiveIndexWriterConfig.java Thu Aug 1 18:42:11 2013
@@ -160,8 +160,9 @@ public class LiveIndexWriterConfig {
}
/**
- * Determines the minimal number of delete terms required before the buffered
- * in-memory delete terms and queries are applied and flushed.
+ * Determines the maximum number of delete-by-term operations that will be
+ * buffered before both the buffered in-memory delete terms and queries are
+ * applied and flushed.
* <p>
* Disabled by default (writer flushes by RAM usage).
* <p>
@@ -169,7 +170,8 @@ public class LiveIndexWriterConfig {
*
* <p>
* Takes effect immediately, but only the next time a document is added,
- * updated or deleted.
+ * updated or deleted. Also, if you only delete-by-query, this setting has no
+ * effect, i.e. delete queries are buffered until the next segment is flushed.
*
* @throws IllegalArgumentException
* if maxBufferedDeleteTerms is enabled but smaller than 1
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1509380&r1=1509379&r2=1509380&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SegmentWriteState.java Thu Aug 1 18:42:11 2013
@@ -83,9 +83,7 @@ public class SegmentWriteState {
this.context = context;
}
- /**
- * Create a shallow {@link SegmentWriteState} copy final a format ID
- */
+ /** Create a shallow copy of {@link SegmentWriteState} with a new segment suffix. */
public SegmentWriteState(SegmentWriteState state, String segmentSuffix) {
infoStream = state.infoStream;
directory = state.directory;