You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2011/11/14 23:36:32 UTC
svn commit: r1201946 [3/14] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/
luce...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java Mon Nov 14 22:36:20 2011
@@ -19,12 +19,10 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
-import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -52,6 +50,7 @@ import org.apache.lucene.store.MergeInfo
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.MapBackedSet;
@@ -102,11 +101,6 @@ import org.apache.lucene.util.TwoPhaseCo
addDocument calls (see <a href="#mergePolicy">below</a>
for changing the {@link MergeScheduler}).</p>
- <p>If an index will not have more documents added for a while and optimal search
- performance is desired, then either the full {@link #optimize() optimize}
- method or partial {@link #optimize(int)} method should be
- called before the index is closed.</p>
-
<p>Opening an <code>IndexWriter</code> creates a lock file for the directory in use. Trying to open
another <code>IndexWriter</code> on the same directory will lead to a
{@link LockObtainFailedException}. The {@link LockObtainFailedException}
@@ -135,9 +129,8 @@ import org.apache.lucene.util.TwoPhaseCo
The {@link MergePolicy} is invoked whenever there are
changes to the segments in the index. Its role is to
select which merges to do, if any, and return a {@link
- MergePolicy.MergeSpecification} describing the merges. It
- also selects merges to do for optimize(). (The default is
- {@link LogByteSizeMergePolicy}. Then, the {@link
+ MergePolicy.MergeSpecification} describing the merges.
+ The default is {@link LogByteSizeMergePolicy}. Then, the {@link
MergeScheduler} is invoked with the requested merges and
it decides when and how to run the merges. The default is
{@link ConcurrentMergeScheduler}. </p>
@@ -202,12 +195,9 @@ public class IndexWriter implements Clos
* encoded as UTF8. If a term arrives from the analyzer
* longer than this length, it is skipped and a message is
* printed to infoStream, if set (see {@link
- * #setInfoStream}).
+ * IndexWriterConfig#setInfoStream(InfoStream)}).
*/
public final static int MAX_TERM_LENGTH = DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8;
- // Used for printing messages
- private static final AtomicInteger MESSAGE_ID = new AtomicInteger();
- private int messageID = MESSAGE_ID.getAndIncrement();
volatile private boolean hitOOM;
private final Directory directory; // where this index resides
@@ -227,8 +217,9 @@ public class IndexWriter implements Clos
private DocumentsWriter docWriter;
final IndexFileDeleter deleter;
- private Map<SegmentInfo,Boolean> segmentsToOptimize = new HashMap<SegmentInfo,Boolean>(); // used by optimize to note those needing optimization
- private int optimizeMaxNumSegments;
+ // used by forceMerge to note those needing merging
+ private Map<SegmentInfo,Boolean> segmentsToMerge = new HashMap<SegmentInfo,Boolean>();
+ private int mergeMaxNumSegments;
private Lock writeLock;
@@ -271,9 +262,6 @@ public class IndexWriter implements Clos
// The PayloadProcessorProvider to use when segments are merged
private PayloadProcessorProvider payloadProcessorProvider;
- // for testing
- boolean anyNonBulkMerges;
-
IndexReader getReader() throws IOException {
return getReader(true);
}
@@ -343,7 +331,7 @@ public class IndexWriter implements Clos
final long tStart = System.currentTimeMillis();
if (infoStream != null) {
- message("flush at getReader");
+ infoStream.message("IW", "flush at getReader");
}
// Do this up front before flushing so that the readers
// obtained during this flush are pooled, the first time
@@ -376,7 +364,7 @@ public class IndexWriter implements Clos
maybeApplyDeletes(applyAllDeletes);
r = new DirectoryReader(this, segmentInfos, applyAllDeletes);
if (infoStream != null) {
- message("return reader version=" + r.getVersion() + " reader=" + r);
+ infoStream.message("IW", "return reader version=" + r.getVersion() + " reader=" + r);
}
}
} catch (OutOfMemoryError oom) {
@@ -385,7 +373,7 @@ public class IndexWriter implements Clos
return null;
} finally {
if (!success && infoStream != null) {
- message("hit exception during NRT reader");
+ infoStream.message("IW", "hit exception during NRT reader");
}
// Done: finish the full flush!
docWriter.finishFullFlush(success);
@@ -396,7 +384,7 @@ public class IndexWriter implements Clos
maybeMerge();
}
if (infoStream != null) {
- message("getReader took " + (System.currentTimeMillis() - tStart) + " msec");
+ infoStream.message("IW", "getReader took " + (System.currentTimeMillis() - tStart) + " msec");
}
return r;
}
@@ -791,16 +779,6 @@ public class IndexWriter implements Clos
ensureOpen(true);
}
- /**
- * Prints a message to the infoStream (if non-null),
- * prefixed with the identifying information for this
- * writer and the thread that's calling it.
- */
- public void message(String message) {
- if (infoStream != null)
- infoStream.println("IW " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message);
- }
-
final Codec codec; // for writing new segments
/**
@@ -832,14 +810,13 @@ public class IndexWriter implements Clos
config = (IndexWriterConfig) conf.clone();
directory = d;
analyzer = conf.getAnalyzer();
- infoStream = defaultInfoStream;
+ infoStream = conf.getInfoStream();
mergePolicy = conf.getMergePolicy();
mergePolicy.setIndexWriter(this);
mergeScheduler = conf.getMergeScheduler();
codec = conf.getCodec();
- bufferedDeletesStream = new BufferedDeletesStream(messageID);
- bufferedDeletesStream.setInfoStream(infoStream);
+ bufferedDeletesStream = new BufferedDeletesStream(infoStream);
poolReaders = conf.getReaderPooling();
writeLock = directory.makeLock(WRITE_LOCK_NAME);
@@ -897,7 +874,7 @@ public class IndexWriter implements Clos
changeCount++;
segmentInfos.changed();
if (infoStream != null)
- message("init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
+ infoStream.message("IW", "init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
}
}
@@ -906,7 +883,6 @@ public class IndexWriter implements Clos
// start with previous field numbers, but new FieldInfos
globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory);
docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
- docWriter.setInfoStream(infoStream);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
@@ -926,7 +902,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("init: create=" + create);
+ infoStream.message("IW", "init: create=" + create);
messageState();
}
@@ -935,7 +911,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("init: hit exception on init; releasing write lock");
+ infoStream.message("IW", "init: hit exception on init; releasing write lock");
}
try {
writeLock.release();
@@ -963,51 +939,13 @@ public class IndexWriter implements Clos
return config;
}
- /** If non-null, this will be the default infoStream used
- * by a newly instantiated IndexWriter.
- * @see #setInfoStream
- */
- public static void setDefaultInfoStream(PrintStream infoStream) {
- IndexWriter.defaultInfoStream = infoStream;
- }
-
- /**
- * Returns the current default infoStream for newly
- * instantiated IndexWriters.
- * @see #setDefaultInfoStream
- */
- public static PrintStream getDefaultInfoStream() {
- return IndexWriter.defaultInfoStream;
- }
-
- /** If non-null, information about merges, deletes and a
- * message when maxFieldLength is reached will be printed
- * to this.
- */
- public void setInfoStream(PrintStream infoStream) throws IOException {
- ensureOpen();
- this.infoStream = infoStream;
- docWriter.setInfoStream(infoStream);
- deleter.setInfoStream(infoStream);
- bufferedDeletesStream.setInfoStream(infoStream);
- if (infoStream != null)
- messageState();
- }
-
private void messageState() throws IOException {
- message("\ndir=" + directory + "\n" +
+ if (infoStream != null) {
+ infoStream.message("IW", "\ndir=" + directory + "\n" +
"index=" + segString() + "\n" +
"version=" + Constants.LUCENE_VERSION + "\n" +
config.toString());
- }
-
- /**
- * Returns the current infoStream in use by this writer.
- * @see #setInfoStream
- */
- public PrintStream getInfoStream() {
- ensureOpen();
- return infoStream;
+ }
}
/** Returns true if verbosing is enabled (i.e., infoStream != null). */
@@ -1122,7 +1060,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message("now flush at close waitForMerges=" + waitForMerges);
+ infoStream.message("IW", "now flush at close waitForMerges=" + waitForMerges);
}
docWriter.close();
@@ -1148,14 +1086,14 @@ public class IndexWriter implements Clos
mergeScheduler.close();
if (infoStream != null)
- message("now call final commit()");
+ infoStream.message("IW", "now call final commit()");
if (!hitOOM) {
commitInternal(null);
}
if (infoStream != null)
- message("at close: " + segString());
+ infoStream.message("IW", "at close: " + segString());
// used by assert below
final DocumentsWriter oldWriter = docWriter;
synchronized(this) {
@@ -1180,7 +1118,7 @@ public class IndexWriter implements Clos
notifyAll();
if (!closed) {
if (infoStream != null)
- message("hit exception while closing");
+ infoStream.message("IW", "hit exception while closing");
}
}
}
@@ -1272,7 +1210,7 @@ public class IndexWriter implements Clos
* readers/searchers are open against the index, and up to
* 2X the size of all segments being merged when
* readers/searchers are open against the index (see
- * {@link #optimize()} for details). The sequence of
+ * {@link #forceMerge(int)} for details). The sequence of
* primitive merge operations performed is governed by the
* merge policy.
*
@@ -1410,7 +1348,7 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success && infoStream != null) {
- message("hit exception updating document");
+ infoStream.message("IW", "hit exception updating document");
}
}
if (anySegmentFlushed) {
@@ -1557,7 +1495,7 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success && infoStream != null)
- message("hit exception updating document");
+ infoStream.message("IW", "hit exception updating document");
}
if (anySegmentFlushed) {
@@ -1619,59 +1557,55 @@ public class IndexWriter implements Clos
/** If non-null, information about merges will be printed to this.
*/
- private PrintStream infoStream;
- private static PrintStream defaultInfoStream;
+ final InfoStream infoStream;
/**
- * Requests an "optimize" operation on an index, priming the index
- * for the fastest available search. Traditionally this has meant
- * merging all segments into a single segment as is done in the
- * default merge policy, but individual merge policies may implement
- * optimize in different ways.
- *
- * <p> Optimize is a very costly operation, so you
- * should only do it if your search performance really
- * requires it. Many search applications do fine never
- * calling optimize. </p>
+ * Forces merge policy to merge segments until there's <=
+ * maxNumSegments. The actual merges to be
+ * executed are determined by the {@link MergePolicy}.
+ *
+ * <p>This is a horribly costly operation, especially when
+ * you pass a small {@code maxNumSegments}; usually you
+ * should only call this if the index is static (will no
+ * longer be changed).</p>
*
- * <p>Note that optimize requires 2X the index size free
+ * <p>Note that this requires up to 2X the index size free
* space in your Directory (3X if you're using compound
* file format). For example, if your index size is 10 MB
- * then you need 20 MB free for optimize to complete (30
+ * then you need up to 20 MB free for this to complete (30
* MB if you're using compound file format). Also,
- * it's best to call {@link #commit()} after the optimize
- * completes to allow IndexWriter to free up disk space.</p>
+ * it's best to call {@link #commit()} afterwards,
+ * to allow IndexWriter to free up disk space.</p>
*
- * <p>If some but not all readers re-open while an
- * optimize is underway, this will cause > 2X temporary
+ * <p>If some but not all readers re-open while merging
+ * is underway, this will cause > 2X temporary
* space to be consumed as those new readers will then
- * hold open the partially optimized segments at that
- * time. It is best not to re-open readers while optimize
- * is running.</p>
+ * hold open the temporary segments at that time. It is
+ * best not to re-open readers while merging is running.</p>
*
* <p>The actual temporary usage could be much less than
* these figures (it depends on many factors).</p>
*
- * <p>In general, once the optimize completes, the total size of the
+ * <p>In general, once the this completes, the total size of the
* index will be less than the size of the starting index.
* It could be quite a bit smaller (if there were many
* pending deletes) or just slightly smaller.</p>
*
- * <p>If an Exception is hit during optimize(), for example
+ * <p>If an Exception is hit, for example
* due to disk full, the index will not be corrupt and no
* documents will have been lost. However, it may have
- * been partially optimized (some segments were merged but
+ * been partially merged (some segments were merged but
* not all), and it's possible that one of the segments in
* the index will be in non-compound format even when
* using compound file format. This will occur when the
* Exception is hit during conversion of the segment into
* compound format.</p>
*
- * <p>This call will optimize those segments present in
+ * <p>This call will merge those segments present in
* the index when the call started. If other threads are
* still adding documents and flushing segments, those
- * newly created segments will not be optimized unless you
- * call optimize again.</p>
+ * newly created segments will not be merged unless you
+ * call forceMerge again.</p>
*
* <p><b>NOTE</b>: if this method hits an OutOfMemoryError
* you should immediately close the writer. See <a
@@ -1684,96 +1618,66 @@ public class IndexWriter implements Clos
*
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
- * @see MergePolicy#findMergesForOptimize
- */
- public void optimize() throws CorruptIndexException, IOException {
- optimize(true);
- }
-
- /**
- * Optimize the index down to <= maxNumSegments. If
- * maxNumSegments==1 then this is the same as {@link
- * #optimize()}.
- *
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
+ * @see MergePolicy#findMerges
*
* @param maxNumSegments maximum number of segments left
- * in the index after optimization finishes
- */
- public void optimize(int maxNumSegments) throws CorruptIndexException, IOException {
- optimize(maxNumSegments, true);
- }
-
- /** Just like {@link #optimize()}, except you can specify
- * whether the call should block until the optimize
- * completes. This is only meaningful with a
- * {@link MergeScheduler} that is able to run merges in
- * background threads.
- *
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
- */
- public void optimize(boolean doWait) throws CorruptIndexException, IOException {
- optimize(1, doWait);
+ * in the index after merging finishes
+ */
+ public void forceMerge(int maxNumSegments) throws CorruptIndexException, IOException {
+ forceMerge(maxNumSegments, true);
}
- /** Just like {@link #optimize(int)}, except you can
- * specify whether the call should block until the
- * optimize completes. This is only meaningful with a
+ /** Just like {@link #forceMerge(int)}, except you can
+ * specify whether the call should block until
+ * all merging completes. This is only meaningful with a
* {@link MergeScheduler} that is able to run merges in
* background threads.
*
- * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
- * you should immediately close the writer. See <a
- * href="#OOME">above</a> for details.</p>
+ * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
+ * you should immediately close the writer. See <a
+ * href="#OOME">above</a> for details.</p>
*/
- public void optimize(int maxNumSegments, boolean doWait) throws CorruptIndexException, IOException {
+ public void forceMerge(int maxNumSegments, boolean doWait) throws CorruptIndexException, IOException {
ensureOpen();
if (maxNumSegments < 1)
throw new IllegalArgumentException("maxNumSegments must be >= 1; got " + maxNumSegments);
if (infoStream != null) {
- message("optimize: index now " + segString());
- message("now flush at optimize");
+ infoStream.message("IW", "forceMerge: index now " + segString());
+ infoStream.message("IW", "now flush at forceMerge");
}
flush(true, true);
synchronized(this) {
resetMergeExceptions();
- segmentsToOptimize.clear();
+ segmentsToMerge.clear();
for(SegmentInfo info : segmentInfos) {
- segmentsToOptimize.put(info, Boolean.TRUE);
+ segmentsToMerge.put(info, Boolean.TRUE);
}
- optimizeMaxNumSegments = maxNumSegments;
+ mergeMaxNumSegments = maxNumSegments;
- // Now mark all pending & running merges as optimize
- // merge:
+ // Now mark all pending & running merges as isMaxNumSegments:
for(final MergePolicy.OneMerge merge : pendingMerges) {
- merge.optimize = true;
- merge.maxNumSegmentsOptimize = maxNumSegments;
- segmentsToOptimize.put(merge.info, Boolean.TRUE);
+ merge.maxNumSegments = maxNumSegments;
+ segmentsToMerge.put(merge.info, Boolean.TRUE);
}
for ( final MergePolicy.OneMerge merge: runningMerges ) {
- merge.optimize = true;
- merge.maxNumSegmentsOptimize = maxNumSegments;
- segmentsToOptimize.put(merge.info, Boolean.TRUE);
+ merge.maxNumSegments = maxNumSegments;
+ segmentsToMerge.put(merge.info, Boolean.TRUE);
}
}
- maybeMerge(maxNumSegments, true);
+ maybeMerge(maxNumSegments);
if (doWait) {
synchronized(this) {
while(true) {
if (hitOOM) {
- throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot complete optimize");
+ throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot complete forceMerge");
}
if (mergeExceptions.size() > 0) {
@@ -1782,7 +1686,7 @@ public class IndexWriter implements Clos
final int size = mergeExceptions.size();
for(int i=0;i<size;i++) {
final MergePolicy.OneMerge merge = mergeExceptions.get(i);
- if (merge.optimize) {
+ if (merge.maxNumSegments != -1) {
IOException err = new IOException("background merge hit exception: " + merge.segString(directory));
final Throwable t = merge.getException();
if (t != null)
@@ -1792,7 +1696,7 @@ public class IndexWriter implements Clos
}
}
- if (optimizeMergesPending())
+ if (maxNumSegmentsMergesPending())
doWait();
else
break;
@@ -1801,26 +1705,26 @@ public class IndexWriter implements Clos
// If close is called while we are still
// running, throw an exception so the calling
- // thread will know the optimize did not
+ // thread will know merging did not
// complete
ensureOpen();
}
// NOTE: in the ConcurrentMergeScheduler case, when
// doWait is false, we can return immediately while
- // background threads accomplish the optimization
+ // background threads accomplish the merging
}
/** Returns true if any merges in pendingMerges or
- * runningMerges are optimization merges. */
- private synchronized boolean optimizeMergesPending() {
+ * runningMerges are maxNumSegments merges. */
+ private synchronized boolean maxNumSegmentsMergesPending() {
for (final MergePolicy.OneMerge merge : pendingMerges) {
- if (merge.optimize)
+ if (merge.maxNumSegments != -1)
return true;
}
for (final MergePolicy.OneMerge merge : runningMerges) {
- if (merge.optimize)
+ if (merge.maxNumSegments != -1)
return true;
}
@@ -1849,7 +1753,7 @@ public class IndexWriter implements Clos
flush(true, true);
if (infoStream != null)
- message("expungeDeletes: index now " + segString());
+ infoStream.message("IW", "expungeDeletes: index now " + segString());
MergePolicy.MergeSpecification spec;
@@ -1899,7 +1803,7 @@ public class IndexWriter implements Clos
// NOTE: in the ConcurrentMergeScheduler case, when
// doWait is false, we can return immediately while
- // background threads accomplish the optimization
+ // background threads accomplish the merging
}
@@ -1912,14 +1816,14 @@ public class IndexWriter implements Clos
*
* <p>When an index
* has many document deletions (or updates to existing
- * documents), it's best to either call optimize or
+ * documents), it's best to either call forceMerge or
* expungeDeletes to remove all unused data in the index
* associated with the deleted documents. To see how
* many deletions you have pending in your index, call
* {@link IndexReader#numDeletedDocs}
* This saves disk space and memory usage while
* searching. expungeDeletes should be somewhat faster
- * than optimize since it does not insist on reducing the
+ * than forceMerge since it does not insist on reducing the
* index to a single segment (though, this depends on the
* {@link MergePolicy}; see {@link
* MergePolicy#findMergesToExpungeDeletes}.). Note that
@@ -1954,22 +1858,18 @@ public class IndexWriter implements Clos
* href="#OOME">above</a> for details.</p>
*/
public final void maybeMerge() throws CorruptIndexException, IOException {
- maybeMerge(false);
+ maybeMerge(-1);
}
- private final void maybeMerge(boolean optimize) throws CorruptIndexException, IOException {
- maybeMerge(1, optimize);
- }
-
- private final void maybeMerge(int maxNumSegmentsOptimize, boolean optimize) throws CorruptIndexException, IOException {
+ private final void maybeMerge(int maxNumSegments) throws CorruptIndexException, IOException {
ensureOpen(false);
- updatePendingMerges(maxNumSegmentsOptimize, optimize);
+ updatePendingMerges(maxNumSegments);
mergeScheduler.merge(this);
}
- private synchronized void updatePendingMerges(int maxNumSegmentsOptimize, boolean optimize)
+ private synchronized void updatePendingMerges(int maxNumSegments)
throws CorruptIndexException, IOException {
- assert !optimize || maxNumSegmentsOptimize > 0;
+ assert maxNumSegments == -1 || maxNumSegments > 0;
if (stopMerges) {
return;
@@ -1981,14 +1881,13 @@ public class IndexWriter implements Clos
}
final MergePolicy.MergeSpecification spec;
- if (optimize) {
- spec = mergePolicy.findMergesForOptimize(segmentInfos, maxNumSegmentsOptimize, Collections.unmodifiableMap(segmentsToOptimize));
+ if (maxNumSegments != -1) {
+ spec = mergePolicy.findForcedMerges(segmentInfos, maxNumSegments, Collections.unmodifiableMap(segmentsToMerge));
if (spec != null) {
final int numMerges = spec.merges.size();
for(int i=0;i<numMerges;i++) {
final MergePolicy.OneMerge merge = spec.merges.get(i);
- merge.optimize = true;
- merge.maxNumSegmentsOptimize = maxNumSegmentsOptimize;
+ merge.maxNumSegments = maxNumSegments;
}
}
@@ -2057,7 +1956,7 @@ public class IndexWriter implements Clos
boolean success = false;
if (infoStream != null ) {
- message("rollback");
+ infoStream.message("IW", "rollback");
}
try {
@@ -2067,7 +1966,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null ) {
- message("rollback: done finish merges");
+ infoStream.message("IW", "rollback: done finish merges");
}
// Must pre-close these two, in case they increment
@@ -2094,7 +1993,7 @@ public class IndexWriter implements Clos
// once").
segmentInfos.rollbackSegmentInfos(rollbackSegments);
if (infoStream != null ) {
- message("rollback: infos=" + segString(segmentInfos));
+ infoStream.message("IW", "rollback: infos=" + segString(segmentInfos));
}
docWriter.abort();
@@ -2121,7 +2020,7 @@ public class IndexWriter implements Clos
closing = false;
notifyAll();
if (infoStream != null)
- message("hit exception during rollback");
+ infoStream.message("IW", "hit exception during rollback");
}
}
}
@@ -2141,7 +2040,7 @@ public class IndexWriter implements Clos
*
* <p>NOTE: this method will forcefully abort all merges
* in progress. If other threads are running {@link
- * #optimize()}, {@link #addIndexes(IndexReader[])} or
+ * #forceMerge}, {@link #addIndexes(IndexReader[])} or
* {@link #expungeDeletes} methods, they may receive
* {@link MergePolicy.MergeAbortedException}s.
*/
@@ -2174,7 +2073,7 @@ public class IndexWriter implements Clos
handleOOM(oom, "deleteAll");
} finally {
if (!success && infoStream != null) {
- message("hit exception during deleteAll");
+ infoStream.message("IW", "hit exception during deleteAll");
}
}
}
@@ -2187,7 +2086,7 @@ public class IndexWriter implements Clos
// Abort all pending & running merges:
for (final MergePolicy.OneMerge merge : pendingMerges) {
if (infoStream != null)
- message("now abort pending merge " + merge.segString(directory));
+ infoStream.message("IW", "now abort pending merge " + merge.segString(directory));
merge.abort();
mergeFinish(merge);
}
@@ -2195,7 +2094,7 @@ public class IndexWriter implements Clos
for (final MergePolicy.OneMerge merge : runningMerges) {
if (infoStream != null)
- message("now abort running merge " + merge.segString(directory));
+ infoStream.message("IW", "now abort running merge " + merge.segString(directory));
merge.abort();
}
@@ -2206,7 +2105,7 @@ public class IndexWriter implements Clos
// they are aborted.
while(runningMerges.size() > 0) {
if (infoStream != null)
- message("now wait for " + runningMerges.size() + " running merge to abort");
+ infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge to abort");
doWait();
}
@@ -2216,7 +2115,7 @@ public class IndexWriter implements Clos
assert 0 == mergingSegments.size();
if (infoStream != null)
- message("all running merges have aborted");
+ infoStream.message("IW", "all running merges have aborted");
} else {
// waitForMerges() will ensure any running addIndexes finishes.
@@ -2237,7 +2136,7 @@ public class IndexWriter implements Clos
public synchronized void waitForMerges() {
ensureOpen(false);
if (infoStream != null) {
- message("waitForMerges");
+ infoStream.message("IW", "waitForMerges");
}
while(pendingMerges.size() > 0 || runningMerges.size() > 0) {
doWait();
@@ -2247,7 +2146,7 @@ public class IndexWriter implements Clos
assert 0 == mergingSegments.size();
if (infoStream != null) {
- message("waitForMerges done");
+ infoStream.message("IW", "waitForMerges done");
}
}
@@ -2284,7 +2183,9 @@ public class IndexWriter implements Clos
try {
if (useCompoundFile(newSegment)) {
String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
- message("creating compound file " + compoundFileName);
+ if (infoStream != null) {
+ infoStream.message("IW", "creating compound file " + compoundFileName);
+ }
// Now build compound file
final Directory cfsDir = new CompoundFileDirectory(directory, compoundFileName, context, true);
IOException prior = null;
@@ -2315,7 +2216,7 @@ public class IndexWriter implements Clos
newSegment.advanceDelGen();
final String delFileName = newSegment.getDelFileName();
if (infoStream != null) {
- message("flush: write " + delCount + " deletes to " + delFileName);
+ infoStream.message("IW", "flush: write " + delCount + " deletes to " + delFileName);
}
boolean success2 = false;
try {
@@ -2342,7 +2243,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("hit exception " +
+ infoStream.message("IW", "hit exception " +
"reating compound file for newly flushed segment " + newSegment.name);
}
@@ -2374,7 +2275,7 @@ public class IndexWriter implements Clos
// Lock order IW -> BDS
synchronized (bufferedDeletesStream) {
if (infoStream != null) {
- message("publishFlushedSegment");
+ infoStream.message("IW", "publishFlushedSegment");
}
if (globalPacket != null && globalPacket.any()) {
@@ -2391,7 +2292,7 @@ public class IndexWriter implements Clos
nextGen = bufferedDeletesStream.getNextGen();
}
if (infoStream != null) {
- message("publish sets newSegment delGen=" + nextGen);
+ infoStream.message("IW", "publish sets newSegment delGen=" + nextGen);
}
newSegment.setBufferedDeletesGen(nextGen);
segmentInfos.add(newSegment);
@@ -2446,7 +2347,7 @@ public class IndexWriter implements Clos
* (including the starting index). If readers/searchers
* are open against the starting index, then temporary
* free space required will be higher by the size of the
- * starting index (see {@link #optimize()} for details).
+ * starting index (see {@link #forceMerge(int)} for details).
*
* <p>
* <b>NOTE:</b> this method only copies the segments of the incoming indexes
@@ -2473,7 +2374,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null)
- message("flush at addIndexes(Directory...)");
+ infoStream.message("IW", "flush at addIndexes(Directory...)");
flush(false, true);
int docCount = 0;
@@ -2481,7 +2382,7 @@ public class IndexWriter implements Clos
Comparator<String> versionComparator = StringHelper.getVersionComparator();
for (Directory dir : dirs) {
if (infoStream != null) {
- message("addIndexes: process directory " + dir);
+ infoStream.message("IW", "addIndexes: process directory " + dir);
}
SegmentInfos sis = new SegmentInfos(); // read infos from dir
sis.read(dir);
@@ -2495,7 +2396,7 @@ public class IndexWriter implements Clos
String dsName = info.getDocStoreSegment();
if (infoStream != null) {
- message("addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
+ infoStream.message("IW", "addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
}
// create CFS only if the source segment is not CFS, and MP agrees it
@@ -2508,7 +2409,7 @@ public class IndexWriter implements Clos
&& versionComparator.compare(info.getVersion(), "3.1") >= 0;
}
- IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(true), true, false));
+ IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(true), true, -1));
if (createCFS) {
copySegmentIntoCFS(info, newSegName, context);
@@ -2532,7 +2433,6 @@ public class IndexWriter implements Clos
}
/** Merges the provided indexes into this index.
- * <p>After this completes, the index is optimized. </p>
* <p>The provided IndexReaders are not closed.</p>
*
* <p><b>NOTE:</b> while this is running, any attempts to
@@ -2561,28 +2461,28 @@ public class IndexWriter implements Clos
try {
if (infoStream != null)
- message("flush at addIndexes(IndexReader...)");
+ infoStream.message("IW", "flush at addIndexes(IndexReader...)");
flush(false, true);
String mergedName = newSegmentName();
for (IndexReader indexReader : readers) {
numDocs += indexReader.numDocs();
}
- final IOContext context = new IOContext(new MergeInfo(numDocs, -1, true, false));
+ final IOContext context = new IOContext(new MergeInfo(numDocs, -1, true, -1));
// TODO: somehow we should fix this merge so it's
// abortable so that IW.close(false) is able to stop it
- SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(),
- mergedName, null, payloadProcessorProvider,
+ SegmentMerger merger = new SegmentMerger(infoStream, directory, config.getTermIndexInterval(),
+ mergedName, MergeState.CheckAbort.NONE, payloadProcessorProvider,
new FieldInfos(globalFieldNumberMap), codec, context);
for (IndexReader reader : readers) // add new indexes
merger.add(reader);
- int docCount = merger.merge(); // merge 'em
-
- final FieldInfos fieldInfos = merger.fieldInfos();
+ MergeState mergeState = merger.merge(); // merge 'em
+ int docCount = mergeState.mergedDocCount;
+ final FieldInfos fieldInfos = mergeState.fieldInfos;
SegmentInfo info = new SegmentInfo(mergedName, docCount, directory,
- false, merger.getCodec(),
+ false, codec,
fieldInfos);
setDiagnostics(info, "addIndexes(IndexReader...)");
@@ -2753,8 +2653,8 @@ public class IndexWriter implements Clos
ensureOpen(false);
if (infoStream != null) {
- message("prepareCommit: flush");
- message(" index before flush " + segString());
+ infoStream.message("IW", "prepareCommit: flush");
+ infoStream.message("IW", " index before flush " + segString());
}
if (hitOOM) {
@@ -2812,7 +2712,7 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success && infoStream != null) {
- message("hit exception during prepareCommit");
+ infoStream.message("IW", "hit exception during prepareCommit");
}
// Done: finish the full flush!
docWriter.finishFullFlush(flushSuccess);
@@ -2845,7 +2745,7 @@ public class IndexWriter implements Clos
/**
* <p>Commits all pending changes (added & deleted
- * documents, optimizations, segment merges, added
+ * documents, segment merges, added
* indexes, etc.) to the index, and syncs all referenced
* index files, such that a reader will see the changes
* and the index updates will survive an OS or machine
@@ -2896,21 +2796,21 @@ public class IndexWriter implements Clos
private final void commitInternal(Map<String,String> commitUserData) throws CorruptIndexException, IOException {
if (infoStream != null) {
- message("commit: start");
+ infoStream.message("IW", "commit: start");
}
synchronized(commitLock) {
if (infoStream != null) {
- message("commit: enter lock");
+ infoStream.message("IW", "commit: enter lock");
}
if (pendingCommit == null) {
if (infoStream != null) {
- message("commit: now prepare");
+ infoStream.message("IW", "commit: now prepare");
}
prepareCommit(commitUserData);
} else if (infoStream != null) {
- message("commit: already prepared");
+ infoStream.message("IW", "commit: already prepared");
}
finishCommit();
@@ -2922,10 +2822,10 @@ public class IndexWriter implements Clos
if (pendingCommit != null) {
try {
if (infoStream != null)
- message("commit: pendingCommit != null");
+ infoStream.message("IW", "commit: pendingCommit != null");
pendingCommit.finishCommit(directory, codec);
if (infoStream != null)
- message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\"");
+ infoStream.message("IW", "commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\"");
lastCommitChangeCount = pendingCommitChangeCount;
segmentInfos.updateGeneration(pendingCommit);
segmentInfos.setUserData(pendingCommit.getUserData());
@@ -2939,11 +2839,11 @@ public class IndexWriter implements Clos
}
} else if (infoStream != null) {
- message("commit: pendingCommit == null; skip");
+ infoStream.message("IW", "commit: pendingCommit == null; skip");
}
if (infoStream != null) {
- message("commit: done");
+ infoStream.message("IW", "commit: done");
}
}
@@ -2985,8 +2885,8 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message(" start flush: applyAllDeletes=" + applyAllDeletes);
- message(" index before flush " + segString());
+ infoStream.message("IW", " start flush: applyAllDeletes=" + applyAllDeletes);
+ infoStream.message("IW", " index before flush " + segString());
}
final boolean anySegmentFlushed;
@@ -3015,18 +2915,18 @@ public class IndexWriter implements Clos
return false;
} finally {
if (!success && infoStream != null)
- message("hit exception during flush");
+ infoStream.message("IW", "hit exception during flush");
}
}
final synchronized void maybeApplyDeletes(boolean applyAllDeletes) throws IOException {
if (applyAllDeletes) {
if (infoStream != null) {
- message("apply all deletes during flush");
+ infoStream.message("IW", "apply all deletes during flush");
}
applyAllDeletes();
} else if (infoStream != null) {
- message("don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
+ infoStream.message("IW", "don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
}
}
@@ -3039,7 +2939,7 @@ public class IndexWriter implements Clos
}
if (!keepFullyDeletedSegments && result.allDeleted != null) {
if (infoStream != null) {
- message("drop 100% deleted segments: " + segString(result.allDeleted));
+ infoStream.message("IW", "drop 100% deleted segments: " + segString(result.allDeleted));
}
for (SegmentInfo info : result.allDeleted) {
// If a merge has already registered for this
@@ -3104,7 +3004,7 @@ public class IndexWriter implements Clos
final List<SegmentInfo> sourceSegments = merge.segments;
if (infoStream != null)
- message("commitMergeDeletes " + merge.segString(directory));
+ infoStream.message("IW", "commitMergeDeletes " + merge.segString(directory));
// Carefully merge deletes that occurred after we
// started merging:
@@ -3202,7 +3102,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null)
- message("commitMerge: " + merge.segString(directory) + " index=" + segString());
+ infoStream.message("IW", "commitMerge: " + merge.segString(directory) + " index=" + segString());
assert merge.registerDone;
@@ -3214,7 +3114,7 @@ public class IndexWriter implements Clos
// abort this merge
if (merge.isAborted()) {
if (infoStream != null)
- message("commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
+ infoStream.message("IW", "commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
return false;
}
@@ -3230,7 +3130,7 @@ public class IndexWriter implements Clos
final boolean allDeleted = mergedReader.numDocs() == 0;
if (infoStream != null && allDeleted) {
- message("merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
+ infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
}
final boolean dropSegment = allDeleted && !keepFullyDeletedSegments;
@@ -3241,7 +3141,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("after commit: " + segString());
+ infoStream.message("IW", "after commit: " + segString());
}
closeMergeReaders(merge, false);
@@ -3255,10 +3155,10 @@ public class IndexWriter implements Clos
// disk, updating SegmentInfo, etc.:
readerPool.clear(merge.segments);
- if (merge.optimize) {
- // cascade the optimize:
- if (!segmentsToOptimize.containsKey(merge.info)) {
- segmentsToOptimize.put(merge.info, Boolean.FALSE);
+ if (merge.maxNumSegments != -1) {
+ // cascade the forceMerge:
+ if (!segmentsToMerge.containsKey(merge.info)) {
+ segmentsToMerge.put(merge.info, Boolean.FALSE);
}
}
@@ -3268,11 +3168,11 @@ public class IndexWriter implements Clos
final private void handleMergeException(Throwable t, MergePolicy.OneMerge merge) throws IOException {
if (infoStream != null) {
- message("handleMergeException: merge=" + merge.segString(directory) + " exc=" + t);
+ infoStream.message("IW", "handleMergeException: merge=" + merge.segString(directory) + " exc=" + t);
}
// Set the exception on the merge, so if
- // optimize() is waiting on us it sees the root
+ // forceMerge is waiting on us it sees the root
// cause exception:
merge.setException(t);
addMergeException(merge);
@@ -3317,7 +3217,7 @@ public class IndexWriter implements Clos
mergeInit(merge);
if (infoStream != null)
- message("now merge\n merge=" + merge.segString(directory) + "\n index=" + segString());
+ infoStream.message("IW", "now merge\n merge=" + merge.segString(directory) + "\n index=" + segString());
mergeMiddle(merge);
mergeSuccess(merge);
@@ -3331,7 +3231,7 @@ public class IndexWriter implements Clos
if (!success) {
if (infoStream != null)
- message("hit exception during merge");
+ infoStream.message("IW", "hit exception during merge");
if (merge.info != null && !segmentInfos.contains(merge.info))
deleter.refresh(merge.info.name);
}
@@ -3339,8 +3239,8 @@ public class IndexWriter implements Clos
// This merge (and, generally, any change to the
// segments) may now enable new merges, so we call
// merge policy & update pending merges.
- if (success && !merge.isAborted() && (merge.optimize || (!closed && !closing))) {
- updatePendingMerges(merge.maxNumSegmentsOptimize, merge.optimize);
+ if (success && !merge.isAborted() && (merge.maxNumSegments != -1 || (!closed && !closing))) {
+ updatePendingMerges(merge.maxNumSegments);
}
}
}
@@ -3348,7 +3248,7 @@ public class IndexWriter implements Clos
handleOOM(oom, "merge");
}
if (infoStream != null && merge.info != null) {
- message("merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
+ infoStream.message("IW", "merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
}
//System.out.println(Thread.currentThread().getName() + ": merge end");
}
@@ -3384,9 +3284,8 @@ public class IndexWriter implements Clos
if (info.dir != directory) {
isExternal = true;
}
- if (segmentsToOptimize.containsKey(info)) {
- merge.optimize = true;
- merge.maxNumSegmentsOptimize = optimizeMaxNumSegments;
+ if (segmentsToMerge.containsKey(info)) {
+ merge.maxNumSegments = mergeMaxNumSegments;
}
}
@@ -3395,7 +3294,7 @@ public class IndexWriter implements Clos
pendingMerges.add(merge);
if (infoStream != null)
- message("add merge to pendingMerges: " + merge.segString(directory) + " [total " + pendingMerges.size() + " pending]");
+ infoStream.message("IW", "add merge to pendingMerges: " + merge.segString(directory) + " [total " + pendingMerges.size() + " pending]");
merge.mergeGen = mergeGen;
merge.isExternal = isExternal;
@@ -3412,10 +3311,12 @@ public class IndexWriter implements Clos
builder.append("]");
// don't call mergingSegments.toString() could lead to ConcurrentModException
// since merge updates the segments FieldInfos
- message(builder.toString());
+ infoStream.message("IW", builder.toString());
}
for(SegmentInfo info : merge.segments) {
- message("registerMerge info=" + info);
+ if (infoStream != null) {
+ infoStream.message("IW", "registerMerge info=" + info);
+ }
mergingSegments.add(info);
}
@@ -3435,7 +3336,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("hit exception in mergeInit");
+ infoStream.message("IW", "hit exception in mergeInit");
}
mergeFinish(merge);
}
@@ -3447,7 +3348,7 @@ public class IndexWriter implements Clos
assert testPoint("startMergeInit");
assert merge.registerDone;
- assert !merge.optimize || merge.maxNumSegmentsOptimize > 0;
+ assert merge.maxNumSegments == -1 || merge.maxNumSegments > 0;
if (hitOOM) {
throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot merge");
@@ -3477,7 +3378,7 @@ public class IndexWriter implements Clos
if (!keepFullyDeletedSegments && result.allDeleted != null) {
if (infoStream != null) {
- message("drop 100% deleted segments: " + result.allDeleted);
+ infoStream.message("IW", "drop 100% deleted segments: " + result.allDeleted);
}
for(SegmentInfo info : result.allDeleted) {
segmentInfos.remove(info);
@@ -3497,12 +3398,12 @@ public class IndexWriter implements Clos
// Lock order: IW -> BD
bufferedDeletesStream.prune(segmentInfos);
Map<String,String> details = new HashMap<String,String>();
- details.put("optimize", Boolean.toString(merge.optimize));
+ details.put("mergeMaxNumSegments", ""+merge.maxNumSegments);
details.put("mergeFactor", Integer.toString(merge.segments.size()));
setDiagnostics(merge.info, "merge", details);
if (infoStream != null) {
- message("merge seg=" + merge.info.name);
+ infoStream.message("IW", "merge seg=" + merge.info.name);
}
assert merge.estimatedMergeBytes == 0;
@@ -3549,7 +3450,7 @@ public class IndexWriter implements Clos
* the synchronized lock on IndexWriter instance. */
final synchronized void mergeFinish(MergePolicy.OneMerge merge) throws IOException {
- // Optimize, addIndexes or finishMerges may be waiting
+ // forceMerge, addIndexes or finishMerges may be waiting
// on merges to finish.
notifyAll();
@@ -3639,11 +3540,12 @@ public class IndexWriter implements Clos
IOContext context = new IOContext(merge.getMergeInfo());
- SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge,
+ final MergeState.CheckAbort checkAbort = new MergeState.CheckAbort(merge, directory);
+ SegmentMerger merger = new SegmentMerger(infoStream, directory, config.getTermIndexInterval(), mergedName, checkAbort,
payloadProcessorProvider, merge.info.getFieldInfos(), codec, context);
if (infoStream != null) {
- message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors());
+ infoStream.message("IW", "merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors());
}
merge.readers = new ArrayList<SegmentReader>();
@@ -3677,22 +3579,21 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("merge: total " + totDocCount + " docs");
+ infoStream.message("IW", "merge: total " + totDocCount + " docs");
}
merge.checkAborted(directory);
// This is where all the work happens:
- mergedDocCount = merge.info.docCount = merger.merge();
+ MergeState mergeState = merger.merge();
+ mergedDocCount = merge.info.docCount = mergeState.mergedDocCount;
// Record which codec was used to write the segment
- merge.info.setCodec(merger.getCodec());
+ merge.info.setCodec(codec);
if (infoStream != null) {
- message("merge codecs=" + merger.getCodec());
- message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size());
+ infoStream.message("IW", "merge codec=" + codec);
}
- anyNonBulkMerges |= merger.getAnyNonBulkMerges();
assert mergedDocCount == totDocCount: "mergedDocCount=" + mergedDocCount + " vs " + totDocCount;
@@ -3711,7 +3612,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message("create compound file " + compoundFileName);
+ infoStream.message("IW", "create compound file " + compoundFileName);
}
merger.createCompoundFile(compoundFileName, merge.info, new IOContext(merge.getMergeInfo()));
success = true;
@@ -3730,7 +3631,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("hit exception creating compound file during merge");
+ infoStream.message("IW", "hit exception creating compound file during merge");
}
synchronized(this) {
@@ -3751,7 +3652,7 @@ public class IndexWriter implements Clos
if (merge.isAborted()) {
if (infoStream != null) {
- message("abort merge after building CFS");
+ infoStream.message("IW", "abort merge after building CFS");
}
deleter.deleteFile(compoundFileName);
return 0;
@@ -3762,7 +3663,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message(String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes(true)/1024./1024., merge.estimatedMergeBytes/1024/1024.));
+ infoStream.message("IW", String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes(true)/1024./1024., merge.estimatedMergeBytes/1024/1024.));
}
final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
@@ -3936,7 +3837,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message("startCommit(): start");
+ infoStream.message("IW", "startCommit(): start");
}
synchronized(this) {
@@ -3945,14 +3846,14 @@ public class IndexWriter implements Clos
if (pendingCommitChangeCount == lastCommitChangeCount) {
if (infoStream != null) {
- message(" skip startCommit(): no changes pending");
+ infoStream.message("IW", " skip startCommit(): no changes pending");
}
deleter.decRef(toSync);
return;
}
if (infoStream != null) {
- message("startCommit index=" + segString(toSync) + " changeCount=" + changeCount);
+ infoStream.message("IW", "startCommit index=" + segString(toSync) + " changeCount=" + changeCount);
}
assert filesExist(toSync);
@@ -3989,7 +3890,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("done all syncs");
+ infoStream.message("IW", "done all syncs");
}
assert testPoint("midStartCommitSuccess");
@@ -4004,7 +3905,7 @@ public class IndexWriter implements Clos
if (!pendingCommitSet) {
if (infoStream != null) {
- message("hit exception committing segments file");
+ infoStream.message("IW", "hit exception committing segments file");
}
// Hit exception
@@ -4057,7 +3958,7 @@ public class IndexWriter implements Clos
private void handleOOM(OutOfMemoryError oom, String location) {
if (infoStream != null) {
- message("hit OutOfMemoryError inside " + location);
+ infoStream.message("IW", "hit OutOfMemoryError inside " + location);
}
hitOOM = true;
throw oom;
@@ -4082,7 +3983,7 @@ public class IndexWriter implements Clos
//System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any()));
ensureOpen();
if (infoStream != null) {
- message("nrtIsCurrent: infoVersion matches: " + (infos.version == segmentInfos.version) + " DW changes: " + docWriter.anyChanges() + " BD changes: "+bufferedDeletesStream.any());
+ infoStream.message("IW", "nrtIsCurrent: infoVersion matches: " + (infos.version == segmentInfos.version) + " DW changes: " + docWriter.anyChanges() + " BD changes: "+bufferedDeletesStream.any());
}
return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
@@ -4144,7 +4045,7 @@ public class IndexWriter implements Clos
* <b>NOTE:</b> the set {@link PayloadProcessorProvider} will be in effect
* immediately, potentially for already running merges too. If you want to be
* sure it is used for further operations only, such as {@link #addIndexes} or
- * {@link #optimize}, you can call {@link #waitForMerges()} before.
+ * {@link #forceMerge}, you can call {@link #waitForMerges()} before.
*/
public void setPayloadProcessorProvider(PayloadProcessorProvider pcp) {
ensureOpen();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java Mon Nov 14 22:36:20 2011
@@ -17,12 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.PrintStream;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.Version;
/**
@@ -122,6 +126,7 @@ public final class IndexWriterConfig imp
private volatile IndexingChain indexingChain;
private volatile IndexReaderWarmer mergedSegmentWarmer;
private volatile Codec codec;
+ private volatile InfoStream infoStream;
private volatile MergePolicy mergePolicy;
private volatile DocumentsWriterPerThreadPool indexerThreadPool;
private volatile boolean readerPooling;
@@ -159,6 +164,7 @@ public final class IndexWriterConfig imp
indexingChain = DocumentsWriterPerThread.defaultIndexingChain;
mergedSegmentWarmer = null;
codec = Codec.getDefault();
+ infoStream = InfoStream.getDefault();
if (matchVersion.onOrAfter(Version.LUCENE_32)) {
mergePolicy = new TieredMergePolicy();
} else {
@@ -512,7 +518,7 @@ public final class IndexWriterConfig imp
* Expert: {@link MergePolicy} is invoked whenever there are changes to the
* segments in the index. Its role is to select which merges to do, if any,
* and return a {@link MergePolicy.MergeSpecification} describing the merges.
- * It also selects merges to do for optimize(). (The default is
+ * It also selects merges to do for forceMerge. (The default is
* {@link LogByteSizeMergePolicy}.
*
* <p>Only takes effect when IndexWriter is first created. */
@@ -676,6 +682,30 @@ public final class IndexWriterConfig imp
public FlushPolicy getFlushPolicy() {
return flushPolicy;
}
+
+ /**
+ * @see #setInfoStream(InfoStream)
+ */
+ public InfoStream getInfoStream() {
+ return infoStream;
+ }
+
+ /** If non-null, information about merges, deletes and a
+ * message when maxFieldLength is reached will be printed
+ * to this.
+ */
+ public IndexWriterConfig setInfoStream(InfoStream infoStream) {
+ this.infoStream = infoStream;
+ return this;
+ }
+
+ /**
+ * Convenience method that uses {@link PrintStreamInfoStream}
+ */
+ public IndexWriterConfig setInfoStream(PrintStream printStream) {
+ this.infoStream = printStream == null ? null : new PrintStreamInfoStream(printStream);
+ return this;
+ }
@Override
public String toString() {
@@ -695,6 +725,7 @@ public final class IndexWriterConfig imp
sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n");
sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n");
sb.append("codec=").append(codec).append("\n");
+ sb.append("infoStream=").append(infoStream == null ? "null" : infoStream.getClass().getName()).append("\n");
sb.append("mergePolicy=").append(mergePolicy).append("\n");
sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n");
sb.append("readerPooling=").append(readerPooling).append("\n");
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java Mon Nov 14 22:36:20 2011
@@ -31,13 +31,13 @@ public class LogByteSizeMergePolicy exte
public static final double DEFAULT_MAX_MERGE_MB = 2048;
/** Default maximum segment size. A segment of this size
- * or larger will never be merged during optimize. @see setMaxMergeMBForOptimize */
- public static final double DEFAULT_MAX_MERGE_MB_FOR_OPTIMIZE = Long.MAX_VALUE;
+ * or larger will never be merged during forceMerge. @see setMaxMergeMBForForceMerge */
+ public static final double DEFAULT_MAX_MERGE_MB_FOR_FORCED_MERGE = Long.MAX_VALUE;
public LogByteSizeMergePolicy() {
minMergeSize = (long) (DEFAULT_MIN_MERGE_MB*1024*1024);
maxMergeSize = (long) (DEFAULT_MAX_MERGE_MB*1024*1024);
- maxMergeSizeForOptimize = (long) (DEFAULT_MAX_MERGE_MB_FOR_OPTIMIZE*1024*1024);
+ maxMergeSizeForForcedMerge = (long) (DEFAULT_MAX_MERGE_MB_FOR_FORCED_MERGE*1024*1024);
}
@Override
@@ -70,19 +70,19 @@ public class LogByteSizeMergePolicy exte
/** <p>Determines the largest segment (measured by total
* byte size of the segment's files, in MB) that may be
- * merged with other segments during optimize. Setting
+ * merged with other segments during forceMerge. Setting
* it low will leave the index with more than 1 segment,
- * even if {@link IndexWriter#optimize()} is called.*/
- public void setMaxMergeMBForOptimize(double mb) {
- maxMergeSizeForOptimize = (long) (mb*1024*1024);
+ * even if {@link IndexWriter#forceMerge} is called.*/
+ public void setMaxMergeMBForForcedMerge(double mb) {
+ maxMergeSizeForForcedMerge = (long) (mb*1024*1024);
}
/** Returns the largest segment (measured by total byte
* size of the segment's files, in MB) that may be merged
- * with other segments during optimize.
- * @see #setMaxMergeMBForOptimize */
- public double getMaxMergeMBForOptimize() {
- return ((double) maxMergeSizeForOptimize)/1024/1024;
+ * with other segments during forceMerge.
+ * @see #setMaxMergeMBForForcedMerge */
+ public double getMaxMergeMBForForcedMerge() {
+ return ((double) maxMergeSizeForForcedMerge)/1024/1024;
}
/** Sets the minimum size for the lowest level segments.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogDocMergePolicy.java Mon Nov 14 22:36:20 2011
@@ -31,10 +31,10 @@ public class LogDocMergePolicy extends L
public LogDocMergePolicy() {
minMergeSize = DEFAULT_MIN_MERGE_DOCS;
- // maxMergeSize(ForOptimize) are never used by LogDocMergePolicy; set
+ // maxMergeSize(ForForcedMerge) are never used by LogDocMergePolicy; set
// it to Long.MAX_VALUE to disable it
maxMergeSize = Long.MAX_VALUE;
- maxMergeSizeForOptimize = Long.MAX_VALUE;
+ maxMergeSizeForForcedMerge = Long.MAX_VALUE;
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Mon Nov 14 22:36:20 2011
@@ -23,6 +23,8 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.util.InfoStream;
+
/** <p>This class implements a {@link MergePolicy} that tries
* to merge segments into levels of exponentially
* increasing size, where each level has fewer segments than
@@ -68,7 +70,7 @@ public abstract class LogMergePolicy ext
protected long maxMergeSize;
// Although the core MPs set it explicitly, we must default in case someone
// out there wrote his own LMP ...
- protected long maxMergeSizeForOptimize = Long.MAX_VALUE;
+ protected long maxMergeSizeForForcedMerge = Long.MAX_VALUE;
protected int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
protected double noCFSRatio = DEFAULT_NO_CFS_RATIO;
@@ -104,8 +106,12 @@ public abstract class LogMergePolicy ext
}
protected void message(String message) {
- if (verbose())
- writer.get().message("LMP: " + message);
+ if (verbose()) {
+ final InfoStream infoStream = writer.get().infoStream;
+ if (infoStream != null) {
+ infoStream.message("LMP", message);
+ }
+ }
}
/** <p>Returns the number of segments that are merged at
@@ -117,10 +123,10 @@ public abstract class LogMergePolicy ext
/** Determines how often segment indices are merged by
* addDocument(). With smaller values, less RAM is used
- * while indexing, and searches on unoptimized indices are
+ * while indexing, and searches are
* faster, but indexing speed is slower. With larger
* values, more RAM is used during indexing, and while
- * searches on unoptimized indices are slower, indexing is
+ * searches is slower, indexing is
* faster. Thus larger values (> 10) are best for batch
* index creation, and smaller values (< 10) for indices
* that are interactively maintained. */
@@ -201,29 +207,29 @@ public abstract class LogMergePolicy ext
}
}
- protected boolean isOptimized(SegmentInfos infos, int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToOptimize) throws IOException {
+ protected boolean isMerged(SegmentInfos infos, int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToMerge) throws IOException {
final int numSegments = infos.size();
- int numToOptimize = 0;
- SegmentInfo optimizeInfo = null;
+ int numToMerge = 0;
+ SegmentInfo mergeInfo = null;
boolean segmentIsOriginal = false;
- for(int i=0;i<numSegments && numToOptimize <= maxNumSegments;i++) {
+ for(int i=0;i<numSegments && numToMerge <= maxNumSegments;i++) {
final SegmentInfo info = infos.info(i);
- final Boolean isOriginal = segmentsToOptimize.get(info);
+ final Boolean isOriginal = segmentsToMerge.get(info);
if (isOriginal != null) {
segmentIsOriginal = isOriginal;
- numToOptimize++;
- optimizeInfo = info;
+ numToMerge++;
+ mergeInfo = info;
}
}
- return numToOptimize <= maxNumSegments &&
- (numToOptimize != 1 || !segmentIsOriginal || isOptimized(optimizeInfo));
+ return numToMerge <= maxNumSegments &&
+ (numToMerge != 1 || !segmentIsOriginal || isMerged(mergeInfo));
}
- /** Returns true if this single info is optimized (has no
+ /** Returns true if this single info is already fully merged (has no
* pending norms or deletes, is in the same dir as the
* writer, and matches the current compound file setting */
- protected boolean isOptimized(SegmentInfo info)
+ protected boolean isMerged(SegmentInfo info)
throws IOException {
IndexWriter w = writer.get();
assert w != null;
@@ -235,14 +241,14 @@ public abstract class LogMergePolicy ext
}
/**
- * Returns the merges necessary to optimize the index, taking the max merge
+ * Returns the merges necessary to merge the index, taking the max merge
* size or max merge docs into consideration. This method attempts to respect
* the {@code maxNumSegments} parameter, however it might be, due to size
* constraints, that more than that number of segments will remain in the
* index. Also, this method does not guarantee that exactly {@code
* maxNumSegments} will remain, but <= that number.
*/
- private MergeSpecification findMergesForOptimizeSizeLimit(
+ private MergeSpecification findForcedMergesSizeLimit(
SegmentInfos infos, int maxNumSegments, int last) throws IOException {
MergeSpecification spec = new MergeSpecification();
final List<SegmentInfo> segments = infos.asList();
@@ -250,14 +256,15 @@ public abstract class LogMergePolicy ext
int start = last - 1;
while (start >= 0) {
SegmentInfo info = infos.info(start);
- if (size(info) > maxMergeSizeForOptimize || sizeDocs(info) > maxMergeDocs) {
+ if (size(info) > maxMergeSizeForForcedMerge || sizeDocs(info) > maxMergeDocs) {
if (verbose()) {
- message("optimize: skip segment=" + info + ": size is > maxMergeSize (" + maxMergeSizeForOptimize + ") or sizeDocs is > maxMergeDocs (" + maxMergeDocs + ")");
+ message("findForcedMergesSizeLimit: skip segment=" + info + ": size is > maxMergeSize (" + maxMergeSizeForForcedMerge + ") or sizeDocs is > maxMergeDocs (" + maxMergeDocs + ")");
}
// need to skip that segment + add a merge for the 'right' segments,
- // unless there is only 1 which is optimized.
- if (last - start - 1 > 1 || (start != last - 1 && !isOptimized(infos.info(start + 1)))) {
- // there is more than 1 segment to the right of this one, or an unoptimized single segment.
+ // unless there is only 1 which is merged.
+ if (last - start - 1 > 1 || (start != last - 1 && !isMerged(infos.info(start + 1)))) {
+ // there is more than 1 segment to the right of
+ // this one, or a mergeable single segment.
spec.add(new OneMerge(segments.subList(start + 1, last)));
}
last = start;
@@ -269,8 +276,9 @@ public abstract class LogMergePolicy ext
--start;
}
- // Add any left-over segments, unless there is just 1 already optimized.
- if (last > 0 && (++start + 1 < last || !isOptimized(infos.info(start)))) {
+ // Add any left-over segments, unless there is just 1
+ // already fully merged
+ if (last > 0 && (++start + 1 < last || !isMerged(infos.info(start)))) {
spec.add(new OneMerge(segments.subList(start, last)));
}
@@ -278,11 +286,11 @@ public abstract class LogMergePolicy ext
}
/**
- * Returns the merges necessary to optimize the index. This method constraints
+ * Returns the merges necessary to forceMerge the index. This method constraints
* the returned merges only by the {@code maxNumSegments} parameter, and
* guaranteed that exactly that number of segments will remain in the index.
*/
- private MergeSpecification findMergesForOptimizeMaxNumSegments(SegmentInfos infos, int maxNumSegments, int last) throws IOException {
+ private MergeSpecification findForcedMergesMaxNumSegments(SegmentInfos infos, int maxNumSegments, int last) throws IOException {
MergeSpecification spec = new MergeSpecification();
final List<SegmentInfo> segments = infos.asList();
@@ -298,9 +306,9 @@ public abstract class LogMergePolicy ext
if (0 == spec.merges.size()) {
if (maxNumSegments == 1) {
- // Since we must optimize down to 1 segment, the
+ // Since we must merge down to 1 segment, the
// choice is simple:
- if (last > 1 || !isOptimized(infos.info(0))) {
+ if (last > 1 || !isMerged(infos.info(0))) {
spec.add(new OneMerge(segments.subList(0, last)));
}
} else if (last > maxNumSegments) {
@@ -313,7 +321,7 @@ public abstract class LogMergePolicy ext
// We must merge this many segments to leave
// maxNumSegments in the index (from when
- // optimize was first kicked off):
+ // forceMerge was first kicked off):
final int finalMergeSize = last - maxNumSegments + 1;
// Consider all possible starting points:
@@ -336,10 +344,9 @@ public abstract class LogMergePolicy ext
return spec.merges.size() == 0 ? null : spec;
}
- /** Returns the merges necessary to optimize the index.
- * This merge policy defines "optimized" to mean only the
- * requested number of segments is left in the index, and
- * respects the {@link #maxMergeSizeForOptimize} setting.
+ /** Returns the merges necessary to merge the index down
+ * to a specified number of segments.
+ * This respects the {@link #maxMergeSizeForForcedMerge} setting.
* By default, and assuming {@code maxNumSegments=1}, only
* one segment will be left in the index, where that segment
* has no deletions pending nor separate norms, and it is in
@@ -348,30 +355,30 @@ public abstract class LogMergePolicy ext
* (mergeFactor at a time) so the {@link MergeScheduler}
* in use may make use of concurrency. */
@Override
- public MergeSpecification findMergesForOptimize(SegmentInfos infos,
- int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToOptimize) throws IOException {
+ public MergeSpecification findForcedMerges(SegmentInfos infos,
+ int maxNumSegments, Map<SegmentInfo,Boolean> segmentsToMerge) throws IOException {
assert maxNumSegments > 0;
if (verbose()) {
- message("findMergesForOptimize: maxNumSegs=" + maxNumSegments + " segsToOptimize="+ segmentsToOptimize);
+ message("findForcedMerges: maxNumSegs=" + maxNumSegments + " segsToMerge="+ segmentsToMerge);
}
- // If the segments are already optimized (e.g. there's only 1 segment), or
- // there are <maxNumSegements, all optimized, nothing to do.
- if (isOptimized(infos, maxNumSegments, segmentsToOptimize)) {
+ // If the segments are already merged (e.g. there's only 1 segment), or
+ // there are <maxNumSegements:.
+ if (isMerged(infos, maxNumSegments, segmentsToMerge)) {
if (verbose()) {
- message("already optimized; skip");
+ message("already merged; skip");
}
return null;
}
// Find the newest (rightmost) segment that needs to
- // be optimized (other segments may have been flushed
- // since optimize started):
+ // be merged (other segments may have been flushed
+ // since merging started):
int last = infos.size();
while (last > 0) {
final SegmentInfo info = infos.info(--last);
- if (segmentsToOptimize.get(info) != null) {
+ if (segmentsToMerge.get(info) != null) {
last++;
break;
}
@@ -384,8 +391,8 @@ public abstract class LogMergePolicy ext
return null;
}
- // There is only one segment already, and it is optimized
- if (maxNumSegments == 1 && last == 1 && isOptimized(infos.info(0))) {
+ // There is only one segment already, and it is merged
+ if (maxNumSegments == 1 && last == 1 && isMerged(infos.info(0))) {
if (verbose()) {
message("already 1 seg; skip");
}
@@ -396,16 +403,16 @@ public abstract class LogMergePolicy ext
boolean anyTooLarge = false;
for (int i = 0; i < last; i++) {
SegmentInfo info = infos.info(i);
- if (size(info) > maxMergeSizeForOptimize || sizeDocs(info) > maxMergeDocs) {
+ if (size(info) > maxMergeSizeForForcedMerge || sizeDocs(info) > maxMergeDocs) {
anyTooLarge = true;
break;
}
}
if (anyTooLarge) {
- return findMergesForOptimizeSizeLimit(infos, maxNumSegments, last);
+ return findForcedMergesSizeLimit(infos, maxNumSegments, last);
} else {
- return findMergesForOptimizeMaxNumSegments(infos, maxNumSegments, last);
+ return findForcedMergesMaxNumSegments(infos, maxNumSegments, last);
}
}
@@ -655,7 +662,7 @@ public abstract class LogMergePolicy ext
sb.append("minMergeSize=").append(minMergeSize).append(", ");
sb.append("mergeFactor=").append(mergeFactor).append(", ");
sb.append("maxMergeSize=").append(maxMergeSize).append(", ");
- sb.append("maxMergeSizeForOptimize=").append(maxMergeSizeForOptimize).append(", ");
+ sb.append("maxMergeSizeForForcedMerge=").append(maxMergeSizeForForcedMerge).append(", ");
sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", ");
sb.append("maxMergeDocs=").append(maxMergeDocs).append(", ");
sb.append("useCompoundFile=").append(useCompoundFile).append(", ");
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MergePolicy.java Mon Nov 14 22:36:20 2011
@@ -30,8 +30,7 @@ import org.apache.lucene.util.SetOnce;
/**
* <p>Expert: a MergePolicy determines the sequence of
- * primitive merge operations to be used for overall merge
- * and optimize operations.</p>
+ * primitive merge operations.</p>
*
* <p>Whenever the segments in an index have been altered by
* {@link IndexWriter}, either the addition of a newly
@@ -42,8 +41,8 @@ import org.apache.lucene.util.SetOnce;
* merges that are now required. This method returns a
* {@link MergeSpecification} instance describing the set of
* merges that should be done, or null if no merges are
- * necessary. When IndexWriter.optimize is called, it calls
- * {@link #findMergesForOptimize} and the MergePolicy should
+ * necessary. When IndexWriter.forceMerge is called, it calls
+ * {@link #findForcedMerges(SegmentInfos,int,Map)} and the MergePolicy should
* then return the necessary merges.</p>
*
* <p>Note that the policy can return more than one merge at
@@ -69,11 +68,10 @@ public abstract class MergePolicy implem
public static class OneMerge {
SegmentInfo info; // used by IndexWriter
- boolean optimize; // used by IndexWriter
boolean registerDone; // used by IndexWriter
long mergeGen; // used by IndexWriter
boolean isExternal; // used by IndexWriter
- int maxNumSegmentsOptimize; // used by IndexWriter
+ int maxNumSegments = -1; // used by IndexWriter
public long estimatedMergeBytes; // used by IndexWriter
List<SegmentReader> readers; // used by IndexWriter
List<BitVector> readerLiveDocs; // used by IndexWriter
@@ -160,8 +158,8 @@ public abstract class MergePolicy implem
}
if (info != null)
b.append(" into ").append(info.name);
- if (optimize)
- b.append(" [optimize]");
+ if (maxNumSegments != -1)
+ b.append(" [maxNumSegments=" + maxNumSegments + "]");
if (aborted) {
b.append(" [ABORTED]");
}
@@ -193,7 +191,7 @@ public abstract class MergePolicy implem
}
public MergeInfo getMergeInfo() {
- return new MergeInfo(totalDocCount, estimatedMergeBytes, isExternal, optimize);
+ return new MergeInfo(totalDocCount, estimatedMergeBytes, isExternal, maxNumSegments);
}
}
@@ -290,9 +288,9 @@ public abstract class MergePolicy implem
throws CorruptIndexException, IOException;
/**
- * Determine what set of merge operations is necessary in order to optimize
- * the index. {@link IndexWriter} calls this when its
- * {@link IndexWriter#optimize()} method is called. This call is always
+ * Determine what set of merge operations is necessary in
+ * order to merge to <= the specified segment count. {@link IndexWriter} calls this when its
+ * {@link IndexWriter#forceMerge} method is called. This call is always
* synchronized on the {@link IndexWriter} instance so only one thread at a
* time will call this method.
*
@@ -301,17 +299,17 @@ public abstract class MergePolicy implem
* @param maxSegmentCount
* requested maximum number of segments in the index (currently this
* is always 1)
- * @param segmentsToOptimize
+ * @param segmentsToMerge
* contains the specific SegmentInfo instances that must be merged
* away. This may be a subset of all
* SegmentInfos. If the value is True for a
* given SegmentInfo, that means this segment was
* an original segment present in the
- * to-be-optimized index; else, it was a segment
+ * to-be-merged index; else, it was a segment
* produced by a cascaded merge.
*/
- public abstract MergeSpecification findMergesForOptimize(
- SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize)
+ public abstract MergeSpecification findForcedMerges(
+ SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToMerge)
throws CorruptIndexException, IOException;
/**
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiReader.java Mon Nov 14 22:36:20 2011
@@ -234,12 +234,6 @@ public class MultiReader extends IndexRe
}
@Override
- public boolean isOptimized() {
- ensureOpen();
- return false;
- }
-
- @Override
public int numDocs() {
// Don't call ensureOpen() here (it could affect performance)
// NOTE: multiple threads may wind up init'ing
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/NoMergePolicy.java Mon Nov 14 22:36:20 2011
@@ -58,8 +58,8 @@ public final class NoMergePolicy extends
throws CorruptIndexException, IOException { return null; }
@Override
- public MergeSpecification findMergesForOptimize(SegmentInfos segmentInfos,
- int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize)
+ public MergeSpecification findForcedMerges(SegmentInfos segmentInfos,
+ int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToMerge)
throws CorruptIndexException, IOException { return null; }
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/ParallelReader.java Mon Nov 14 22:36:20 2011
@@ -477,23 +477,6 @@ public class ParallelReader extends Inde
return true;
}
- /**
- * Checks recursively if all subindexes are optimized
- */
- @Override
- public boolean isOptimized() {
- ensureOpen();
- for (final IndexReader reader : readers) {
- if (!reader.isOptimized()) {
- return false;
- }
- }
-
- // all subindexes are optimized
- return true;
- }
-
-
/** Not implemented.
* @throws UnsupportedOperationException
*/
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java?rev=1201946&r1=1201945&r2=1201946&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java Mon Nov 14 22:36:20 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.codecs.Pe
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.InfoStream;
/**
* Encapsulates all necessary state to initiate a {@link PerDocConsumer} and
@@ -29,7 +30,7 @@ import org.apache.lucene.util.Counter;
* @lucene.experimental
*/
public class PerDocWriteState {
- public final PrintStream infoStream;
+ public final InfoStream infoStream;
public final Directory directory;
public final String segmentName;
public final FieldInfos fieldInfos;
@@ -37,7 +38,7 @@ public class PerDocWriteState {
public final String segmentSuffix;
public final IOContext context;
- public PerDocWriteState(PrintStream infoStream, Directory directory,
+ public PerDocWriteState(InfoStream infoStream, Directory directory,
String segmentName, FieldInfos fieldInfos, Counter bytesUsed,
String segmentSuffix, IOContext context) {
this.infoStream = infoStream;