You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/09 15:52:38 UTC
svn commit: r1199779 [2/4] - in /lucene/dev/trunk: ./
dev-tools/idea/lucene/contrib/ lucene/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight/
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/
lucene...
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Nov 9 14:52:33 2011
@@ -19,12 +19,10 @@ package org.apache.lucene.index;
import java.io.Closeable;
import java.io.IOException;
-import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
-import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
@@ -52,6 +50,7 @@ import org.apache.lucene.store.MergeInfo
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.MapBackedSet;
@@ -202,12 +201,9 @@ public class IndexWriter implements Clos
* encoded as UTF8. If a term arrives from the analyzer
* longer than this length, it is skipped and a message is
* printed to infoStream, if set (see {@link
- * #setInfoStream}).
+ * IndexWriterConfig#setInfoStream(InfoStream)}).
*/
public final static int MAX_TERM_LENGTH = DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8;
- // Used for printing messages
- private static final AtomicInteger MESSAGE_ID = new AtomicInteger();
- private int messageID = MESSAGE_ID.getAndIncrement();
volatile private boolean hitOOM;
private final Directory directory; // where this index resides
@@ -271,9 +267,6 @@ public class IndexWriter implements Clos
// The PayloadProcessorProvider to use when segments are merged
private PayloadProcessorProvider payloadProcessorProvider;
- // for testing
- boolean anyNonBulkMerges;
-
IndexReader getReader() throws IOException {
return getReader(true);
}
@@ -343,7 +336,7 @@ public class IndexWriter implements Clos
final long tStart = System.currentTimeMillis();
if (infoStream != null) {
- message("flush at getReader");
+ infoStream.message("IW", "flush at getReader");
}
// Do this up front before flushing so that the readers
// obtained during this flush are pooled, the first time
@@ -376,7 +369,7 @@ public class IndexWriter implements Clos
maybeApplyDeletes(applyAllDeletes);
r = new DirectoryReader(this, segmentInfos, applyAllDeletes);
if (infoStream != null) {
- message("return reader version=" + r.getVersion() + " reader=" + r);
+ infoStream.message("IW", "return reader version=" + r.getVersion() + " reader=" + r);
}
}
} catch (OutOfMemoryError oom) {
@@ -385,7 +378,7 @@ public class IndexWriter implements Clos
return null;
} finally {
if (!success && infoStream != null) {
- message("hit exception during NRT reader");
+ infoStream.message("IW", "hit exception during NRT reader");
}
// Done: finish the full flush!
docWriter.finishFullFlush(success);
@@ -396,7 +389,7 @@ public class IndexWriter implements Clos
maybeMerge();
}
if (infoStream != null) {
- message("getReader took " + (System.currentTimeMillis() - tStart) + " msec");
+ infoStream.message("IW", "getReader took " + (System.currentTimeMillis() - tStart) + " msec");
}
return r;
}
@@ -791,16 +784,6 @@ public class IndexWriter implements Clos
ensureOpen(true);
}
- /**
- * Prints a message to the infoStream (if non-null),
- * prefixed with the identifying information for this
- * writer and the thread that's calling it.
- */
- public void message(String message) {
- if (infoStream != null)
- infoStream.println("IW " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message);
- }
-
final Codec codec; // for writing new segments
/**
@@ -832,14 +815,13 @@ public class IndexWriter implements Clos
config = (IndexWriterConfig) conf.clone();
directory = d;
analyzer = conf.getAnalyzer();
- infoStream = defaultInfoStream;
+ infoStream = conf.getInfoStream();
mergePolicy = conf.getMergePolicy();
mergePolicy.setIndexWriter(this);
mergeScheduler = conf.getMergeScheduler();
codec = conf.getCodec();
- bufferedDeletesStream = new BufferedDeletesStream(messageID);
- bufferedDeletesStream.setInfoStream(infoStream);
+ bufferedDeletesStream = new BufferedDeletesStream(infoStream);
poolReaders = conf.getReaderPooling();
writeLock = directory.makeLock(WRITE_LOCK_NAME);
@@ -897,7 +879,7 @@ public class IndexWriter implements Clos
changeCount++;
segmentInfos.changed();
if (infoStream != null)
- message("init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
+ infoStream.message("IW", "init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
}
}
@@ -906,7 +888,6 @@ public class IndexWriter implements Clos
// start with previous field numbers, but new FieldInfos
globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory);
docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
- docWriter.setInfoStream(infoStream);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
@@ -926,7 +907,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("init: create=" + create);
+ infoStream.message("IW", "init: create=" + create);
messageState();
}
@@ -935,7 +916,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("init: hit exception on init; releasing write lock");
+ infoStream.message("IW", "init: hit exception on init; releasing write lock");
}
try {
writeLock.release();
@@ -963,51 +944,13 @@ public class IndexWriter implements Clos
return config;
}
- /** If non-null, this will be the default infoStream used
- * by a newly instantiated IndexWriter.
- * @see #setInfoStream
- */
- public static void setDefaultInfoStream(PrintStream infoStream) {
- IndexWriter.defaultInfoStream = infoStream;
- }
-
- /**
- * Returns the current default infoStream for newly
- * instantiated IndexWriters.
- * @see #setDefaultInfoStream
- */
- public static PrintStream getDefaultInfoStream() {
- return IndexWriter.defaultInfoStream;
- }
-
- /** If non-null, information about merges, deletes and a
- * message when maxFieldLength is reached will be printed
- * to this.
- */
- public void setInfoStream(PrintStream infoStream) throws IOException {
- ensureOpen();
- this.infoStream = infoStream;
- docWriter.setInfoStream(infoStream);
- deleter.setInfoStream(infoStream);
- bufferedDeletesStream.setInfoStream(infoStream);
- if (infoStream != null)
- messageState();
- }
-
private void messageState() throws IOException {
- message("\ndir=" + directory + "\n" +
+ if (infoStream != null) {
+ infoStream.message("IW", "\ndir=" + directory + "\n" +
"index=" + segString() + "\n" +
"version=" + Constants.LUCENE_VERSION + "\n" +
config.toString());
- }
-
- /**
- * Returns the current infoStream in use by this writer.
- * @see #setInfoStream
- */
- public PrintStream getInfoStream() {
- ensureOpen();
- return infoStream;
+ }
}
/** Returns true if verbosing is enabled (i.e., infoStream != null). */
@@ -1122,7 +1065,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message("now flush at close waitForMerges=" + waitForMerges);
+ infoStream.message("IW", "now flush at close waitForMerges=" + waitForMerges);
}
docWriter.close();
@@ -1148,14 +1091,14 @@ public class IndexWriter implements Clos
mergeScheduler.close();
if (infoStream != null)
- message("now call final commit()");
+ infoStream.message("IW", "now call final commit()");
if (!hitOOM) {
commitInternal(null);
}
if (infoStream != null)
- message("at close: " + segString());
+ infoStream.message("IW", "at close: " + segString());
// used by assert below
final DocumentsWriter oldWriter = docWriter;
synchronized(this) {
@@ -1180,7 +1123,7 @@ public class IndexWriter implements Clos
notifyAll();
if (!closed) {
if (infoStream != null)
- message("hit exception while closing");
+ infoStream.message("IW", "hit exception while closing");
}
}
}
@@ -1410,7 +1353,7 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success && infoStream != null) {
- message("hit exception updating document");
+ infoStream.message("IW", "hit exception updating document");
}
}
if (anySegmentFlushed) {
@@ -1557,7 +1500,7 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success && infoStream != null)
- message("hit exception updating document");
+ infoStream.message("IW", "hit exception updating document");
}
if (anySegmentFlushed) {
@@ -1619,8 +1562,7 @@ public class IndexWriter implements Clos
/** If non-null, information about merges will be printed to this.
*/
- private PrintStream infoStream;
- private static PrintStream defaultInfoStream;
+ final InfoStream infoStream;
/**
* Requests an "optimize" operation on an index, priming the index
@@ -1737,8 +1679,8 @@ public class IndexWriter implements Clos
throw new IllegalArgumentException("maxNumSegments must be >= 1; got " + maxNumSegments);
if (infoStream != null) {
- message("optimize: index now " + segString());
- message("now flush at optimize");
+ infoStream.message("IW", "optimize: index now " + segString());
+ infoStream.message("IW", "now flush at optimize");
}
flush(true, true);
@@ -1849,7 +1791,7 @@ public class IndexWriter implements Clos
flush(true, true);
if (infoStream != null)
- message("expungeDeletes: index now " + segString());
+ infoStream.message("IW", "expungeDeletes: index now " + segString());
MergePolicy.MergeSpecification spec;
@@ -2057,7 +1999,7 @@ public class IndexWriter implements Clos
boolean success = false;
if (infoStream != null ) {
- message("rollback");
+ infoStream.message("IW", "rollback");
}
try {
@@ -2067,7 +2009,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null ) {
- message("rollback: done finish merges");
+ infoStream.message("IW", "rollback: done finish merges");
}
// Must pre-close these two, in case they increment
@@ -2094,7 +2036,7 @@ public class IndexWriter implements Clos
// once").
segmentInfos.rollbackSegmentInfos(rollbackSegments);
if (infoStream != null ) {
- message("rollback: infos=" + segString(segmentInfos));
+ infoStream.message("IW", "rollback: infos=" + segString(segmentInfos));
}
docWriter.abort();
@@ -2121,7 +2063,7 @@ public class IndexWriter implements Clos
closing = false;
notifyAll();
if (infoStream != null)
- message("hit exception during rollback");
+ infoStream.message("IW", "hit exception during rollback");
}
}
}
@@ -2174,7 +2116,7 @@ public class IndexWriter implements Clos
handleOOM(oom, "deleteAll");
} finally {
if (!success && infoStream != null) {
- message("hit exception during deleteAll");
+ infoStream.message("IW", "hit exception during deleteAll");
}
}
}
@@ -2187,7 +2129,7 @@ public class IndexWriter implements Clos
// Abort all pending & running merges:
for (final MergePolicy.OneMerge merge : pendingMerges) {
if (infoStream != null)
- message("now abort pending merge " + merge.segString(directory));
+ infoStream.message("IW", "now abort pending merge " + merge.segString(directory));
merge.abort();
mergeFinish(merge);
}
@@ -2195,7 +2137,7 @@ public class IndexWriter implements Clos
for (final MergePolicy.OneMerge merge : runningMerges) {
if (infoStream != null)
- message("now abort running merge " + merge.segString(directory));
+ infoStream.message("IW", "now abort running merge " + merge.segString(directory));
merge.abort();
}
@@ -2206,7 +2148,7 @@ public class IndexWriter implements Clos
// they are aborted.
while(runningMerges.size() > 0) {
if (infoStream != null)
- message("now wait for " + runningMerges.size() + " running merge to abort");
+ infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge to abort");
doWait();
}
@@ -2216,7 +2158,7 @@ public class IndexWriter implements Clos
assert 0 == mergingSegments.size();
if (infoStream != null)
- message("all running merges have aborted");
+ infoStream.message("IW", "all running merges have aborted");
} else {
// waitForMerges() will ensure any running addIndexes finishes.
@@ -2237,7 +2179,7 @@ public class IndexWriter implements Clos
public synchronized void waitForMerges() {
ensureOpen(false);
if (infoStream != null) {
- message("waitForMerges");
+ infoStream.message("IW", "waitForMerges");
}
while(pendingMerges.size() > 0 || runningMerges.size() > 0) {
doWait();
@@ -2247,7 +2189,7 @@ public class IndexWriter implements Clos
assert 0 == mergingSegments.size();
if (infoStream != null) {
- message("waitForMerges done");
+ infoStream.message("IW", "waitForMerges done");
}
}
@@ -2284,7 +2226,9 @@ public class IndexWriter implements Clos
try {
if (useCompoundFile(newSegment)) {
String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
- message("creating compound file " + compoundFileName);
+ if (infoStream != null) {
+ infoStream.message("IW", "creating compound file " + compoundFileName);
+ }
// Now build compound file
final Directory cfsDir = new CompoundFileDirectory(directory, compoundFileName, context, true);
IOException prior = null;
@@ -2315,7 +2259,7 @@ public class IndexWriter implements Clos
newSegment.advanceDelGen();
final String delFileName = newSegment.getDelFileName();
if (infoStream != null) {
- message("flush: write " + delCount + " deletes to " + delFileName);
+ infoStream.message("IW", "flush: write " + delCount + " deletes to " + delFileName);
}
boolean success2 = false;
try {
@@ -2342,7 +2286,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("hit exception " +
+ infoStream.message("IW", "hit exception " +
"reating compound file for newly flushed segment " + newSegment.name);
}
@@ -2374,7 +2318,7 @@ public class IndexWriter implements Clos
// Lock order IW -> BDS
synchronized (bufferedDeletesStream) {
if (infoStream != null) {
- message("publishFlushedSegment");
+ infoStream.message("IW", "publishFlushedSegment");
}
if (globalPacket != null && globalPacket.any()) {
@@ -2391,7 +2335,7 @@ public class IndexWriter implements Clos
nextGen = bufferedDeletesStream.getNextGen();
}
if (infoStream != null) {
- message("publish sets newSegment delGen=" + nextGen);
+ infoStream.message("IW", "publish sets newSegment delGen=" + nextGen);
}
newSegment.setBufferedDeletesGen(nextGen);
segmentInfos.add(newSegment);
@@ -2473,7 +2417,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null)
- message("flush at addIndexes(Directory...)");
+ infoStream.message("IW", "flush at addIndexes(Directory...)");
flush(false, true);
int docCount = 0;
@@ -2481,7 +2425,7 @@ public class IndexWriter implements Clos
Comparator<String> versionComparator = StringHelper.getVersionComparator();
for (Directory dir : dirs) {
if (infoStream != null) {
- message("addIndexes: process directory " + dir);
+ infoStream.message("IW", "addIndexes: process directory " + dir);
}
SegmentInfos sis = new SegmentInfos(); // read infos from dir
sis.read(dir);
@@ -2495,7 +2439,7 @@ public class IndexWriter implements Clos
String dsName = info.getDocStoreSegment();
if (infoStream != null) {
- message("addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
+ infoStream.message("IW", "addIndexes: process segment origName=" + info.name + " newName=" + newSegName + " dsName=" + dsName + " info=" + info);
}
// create CFS only if the source segment is not CFS, and MP agrees it
@@ -2561,7 +2505,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null)
- message("flush at addIndexes(IndexReader...)");
+ infoStream.message("IW", "flush at addIndexes(IndexReader...)");
flush(false, true);
String mergedName = newSegmentName();
@@ -2572,17 +2516,17 @@ public class IndexWriter implements Clos
// TODO: somehow we should fix this merge so it's
// abortable so that IW.close(false) is able to stop it
- SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(),
- mergedName, null, payloadProcessorProvider,
+ SegmentMerger merger = new SegmentMerger(infoStream, directory, config.getTermIndexInterval(),
+ mergedName, MergeState.CheckAbort.NONE, payloadProcessorProvider,
new FieldInfos(globalFieldNumberMap), codec, context);
for (IndexReader reader : readers) // add new indexes
merger.add(reader);
- int docCount = merger.merge(); // merge 'em
-
- final FieldInfos fieldInfos = merger.fieldInfos();
+ MergeState mergeState = merger.merge(); // merge 'em
+ int docCount = mergeState.mergedDocCount;
+ final FieldInfos fieldInfos = mergeState.fieldInfos;
SegmentInfo info = new SegmentInfo(mergedName, docCount, directory,
- false, merger.getCodec(),
+ false, codec,
fieldInfos);
setDiagnostics(info, "addIndexes(IndexReader...)");
@@ -2753,8 +2697,8 @@ public class IndexWriter implements Clos
ensureOpen(false);
if (infoStream != null) {
- message("prepareCommit: flush");
- message(" index before flush " + segString());
+ infoStream.message("IW", "prepareCommit: flush");
+ infoStream.message("IW", " index before flush " + segString());
}
if (hitOOM) {
@@ -2812,7 +2756,7 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success && infoStream != null) {
- message("hit exception during prepareCommit");
+ infoStream.message("IW", "hit exception during prepareCommit");
}
// Done: finish the full flush!
docWriter.finishFullFlush(flushSuccess);
@@ -2896,21 +2840,21 @@ public class IndexWriter implements Clos
private final void commitInternal(Map<String,String> commitUserData) throws CorruptIndexException, IOException {
if (infoStream != null) {
- message("commit: start");
+ infoStream.message("IW", "commit: start");
}
synchronized(commitLock) {
if (infoStream != null) {
- message("commit: enter lock");
+ infoStream.message("IW", "commit: enter lock");
}
if (pendingCommit == null) {
if (infoStream != null) {
- message("commit: now prepare");
+ infoStream.message("IW", "commit: now prepare");
}
prepareCommit(commitUserData);
} else if (infoStream != null) {
- message("commit: already prepared");
+ infoStream.message("IW", "commit: already prepared");
}
finishCommit();
@@ -2922,10 +2866,10 @@ public class IndexWriter implements Clos
if (pendingCommit != null) {
try {
if (infoStream != null)
- message("commit: pendingCommit != null");
+ infoStream.message("IW", "commit: pendingCommit != null");
pendingCommit.finishCommit(directory, codec);
if (infoStream != null)
- message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\"");
+ infoStream.message("IW", "commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\"");
lastCommitChangeCount = pendingCommitChangeCount;
segmentInfos.updateGeneration(pendingCommit);
segmentInfos.setUserData(pendingCommit.getUserData());
@@ -2939,11 +2883,11 @@ public class IndexWriter implements Clos
}
} else if (infoStream != null) {
- message("commit: pendingCommit == null; skip");
+ infoStream.message("IW", "commit: pendingCommit == null; skip");
}
if (infoStream != null) {
- message("commit: done");
+ infoStream.message("IW", "commit: done");
}
}
@@ -2985,8 +2929,8 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message(" start flush: applyAllDeletes=" + applyAllDeletes);
- message(" index before flush " + segString());
+ infoStream.message("IW", " start flush: applyAllDeletes=" + applyAllDeletes);
+ infoStream.message("IW", " index before flush " + segString());
}
final boolean anySegmentFlushed;
@@ -3015,18 +2959,18 @@ public class IndexWriter implements Clos
return false;
} finally {
if (!success && infoStream != null)
- message("hit exception during flush");
+ infoStream.message("IW", "hit exception during flush");
}
}
final synchronized void maybeApplyDeletes(boolean applyAllDeletes) throws IOException {
if (applyAllDeletes) {
if (infoStream != null) {
- message("apply all deletes during flush");
+ infoStream.message("IW", "apply all deletes during flush");
}
applyAllDeletes();
} else if (infoStream != null) {
- message("don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
+ infoStream.message("IW", "don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
}
}
@@ -3039,7 +2983,7 @@ public class IndexWriter implements Clos
}
if (!keepFullyDeletedSegments && result.allDeleted != null) {
if (infoStream != null) {
- message("drop 100% deleted segments: " + segString(result.allDeleted));
+ infoStream.message("IW", "drop 100% deleted segments: " + segString(result.allDeleted));
}
for (SegmentInfo info : result.allDeleted) {
// If a merge has already registered for this
@@ -3104,7 +3048,7 @@ public class IndexWriter implements Clos
final List<SegmentInfo> sourceSegments = merge.segments;
if (infoStream != null)
- message("commitMergeDeletes " + merge.segString(directory));
+ infoStream.message("IW", "commitMergeDeletes " + merge.segString(directory));
// Carefully merge deletes that occurred after we
// started merging:
@@ -3202,7 +3146,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null)
- message("commitMerge: " + merge.segString(directory) + " index=" + segString());
+ infoStream.message("IW", "commitMerge: " + merge.segString(directory) + " index=" + segString());
assert merge.registerDone;
@@ -3214,7 +3158,7 @@ public class IndexWriter implements Clos
// abort this merge
if (merge.isAborted()) {
if (infoStream != null)
- message("commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
+ infoStream.message("IW", "commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
return false;
}
@@ -3230,7 +3174,7 @@ public class IndexWriter implements Clos
final boolean allDeleted = mergedReader.numDocs() == 0;
if (infoStream != null && allDeleted) {
- message("merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
+ infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
}
final boolean dropSegment = allDeleted && !keepFullyDeletedSegments;
@@ -3241,7 +3185,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("after commit: " + segString());
+ infoStream.message("IW", "after commit: " + segString());
}
closeMergeReaders(merge, false);
@@ -3268,7 +3212,7 @@ public class IndexWriter implements Clos
final private void handleMergeException(Throwable t, MergePolicy.OneMerge merge) throws IOException {
if (infoStream != null) {
- message("handleMergeException: merge=" + merge.segString(directory) + " exc=" + t);
+ infoStream.message("IW", "handleMergeException: merge=" + merge.segString(directory) + " exc=" + t);
}
// Set the exception on the merge, so if
@@ -3317,7 +3261,7 @@ public class IndexWriter implements Clos
mergeInit(merge);
if (infoStream != null)
- message("now merge\n merge=" + merge.segString(directory) + "\n index=" + segString());
+ infoStream.message("IW", "now merge\n merge=" + merge.segString(directory) + "\n index=" + segString());
mergeMiddle(merge);
mergeSuccess(merge);
@@ -3331,7 +3275,7 @@ public class IndexWriter implements Clos
if (!success) {
if (infoStream != null)
- message("hit exception during merge");
+ infoStream.message("IW", "hit exception during merge");
if (merge.info != null && !segmentInfos.contains(merge.info))
deleter.refresh(merge.info.name);
}
@@ -3348,7 +3292,7 @@ public class IndexWriter implements Clos
handleOOM(oom, "merge");
}
if (infoStream != null && merge.info != null) {
- message("merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
+ infoStream.message("IW", "merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
}
//System.out.println(Thread.currentThread().getName() + ": merge end");
}
@@ -3395,7 +3339,7 @@ public class IndexWriter implements Clos
pendingMerges.add(merge);
if (infoStream != null)
- message("add merge to pendingMerges: " + merge.segString(directory) + " [total " + pendingMerges.size() + " pending]");
+ infoStream.message("IW", "add merge to pendingMerges: " + merge.segString(directory) + " [total " + pendingMerges.size() + " pending]");
merge.mergeGen = mergeGen;
merge.isExternal = isExternal;
@@ -3412,10 +3356,12 @@ public class IndexWriter implements Clos
builder.append("]");
// don't call mergingSegments.toString() could lead to ConcurrentModException
// since merge updates the segments FieldInfos
- message(builder.toString());
+ infoStream.message("IW", builder.toString());
}
for(SegmentInfo info : merge.segments) {
- message("registerMerge info=" + info);
+ if (infoStream != null) {
+ infoStream.message("IW", "registerMerge info=" + info);
+ }
mergingSegments.add(info);
}
@@ -3435,7 +3381,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("hit exception in mergeInit");
+ infoStream.message("IW", "hit exception in mergeInit");
}
mergeFinish(merge);
}
@@ -3477,7 +3423,7 @@ public class IndexWriter implements Clos
if (!keepFullyDeletedSegments && result.allDeleted != null) {
if (infoStream != null) {
- message("drop 100% deleted segments: " + result.allDeleted);
+ infoStream.message("IW", "drop 100% deleted segments: " + result.allDeleted);
}
for(SegmentInfo info : result.allDeleted) {
segmentInfos.remove(info);
@@ -3502,7 +3448,7 @@ public class IndexWriter implements Clos
setDiagnostics(merge.info, "merge", details);
if (infoStream != null) {
- message("merge seg=" + merge.info.name);
+ infoStream.message("IW", "merge seg=" + merge.info.name);
}
assert merge.estimatedMergeBytes == 0;
@@ -3639,11 +3585,12 @@ public class IndexWriter implements Clos
IOContext context = new IOContext(merge.getMergeInfo());
- SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge,
+ final MergeState.CheckAbort checkAbort = new MergeState.CheckAbort(merge, directory);
+ SegmentMerger merger = new SegmentMerger(infoStream, directory, config.getTermIndexInterval(), mergedName, checkAbort,
payloadProcessorProvider, merge.info.getFieldInfos(), codec, context);
if (infoStream != null) {
- message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors());
+ infoStream.message("IW", "merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors());
}
merge.readers = new ArrayList<SegmentReader>();
@@ -3677,22 +3624,21 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("merge: total " + totDocCount + " docs");
+ infoStream.message("IW", "merge: total " + totDocCount + " docs");
}
merge.checkAborted(directory);
// This is where all the work happens:
- mergedDocCount = merge.info.docCount = merger.merge();
+ MergeState mergeState = merger.merge();
+ mergedDocCount = merge.info.docCount = mergeState.mergedDocCount;
// Record which codec was used to write the segment
- merge.info.setCodec(merger.getCodec());
+ merge.info.setCodec(codec);
if (infoStream != null) {
- message("merge codecs=" + merger.getCodec());
- message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size());
+ infoStream.message("IW", "merge codec=" + codec);
}
- anyNonBulkMerges |= merger.getAnyNonBulkMerges();
assert mergedDocCount == totDocCount: "mergedDocCount=" + mergedDocCount + " vs " + totDocCount;
@@ -3711,7 +3657,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message("create compound file " + compoundFileName);
+ infoStream.message("IW", "create compound file " + compoundFileName);
}
merger.createCompoundFile(compoundFileName, merge.info, new IOContext(merge.getMergeInfo()));
success = true;
@@ -3730,7 +3676,7 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
if (infoStream != null) {
- message("hit exception creating compound file during merge");
+ infoStream.message("IW", "hit exception creating compound file during merge");
}
synchronized(this) {
@@ -3751,7 +3697,7 @@ public class IndexWriter implements Clos
if (merge.isAborted()) {
if (infoStream != null) {
- message("abort merge after building CFS");
+ infoStream.message("IW", "abort merge after building CFS");
}
deleter.deleteFile(compoundFileName);
return 0;
@@ -3762,7 +3708,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message(String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes(true)/1024./1024., merge.estimatedMergeBytes/1024/1024.));
+ infoStream.message("IW", String.format("merged segment size=%.3f MB vs estimate=%.3f MB", merge.info.sizeInBytes(true)/1024./1024., merge.estimatedMergeBytes/1024/1024.));
}
final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
@@ -3936,7 +3882,7 @@ public class IndexWriter implements Clos
try {
if (infoStream != null) {
- message("startCommit(): start");
+ infoStream.message("IW", "startCommit(): start");
}
synchronized(this) {
@@ -3945,14 +3891,14 @@ public class IndexWriter implements Clos
if (pendingCommitChangeCount == lastCommitChangeCount) {
if (infoStream != null) {
- message(" skip startCommit(): no changes pending");
+ infoStream.message("IW", " skip startCommit(): no changes pending");
}
deleter.decRef(toSync);
return;
}
if (infoStream != null) {
- message("startCommit index=" + segString(toSync) + " changeCount=" + changeCount);
+ infoStream.message("IW", "startCommit index=" + segString(toSync) + " changeCount=" + changeCount);
}
assert filesExist(toSync);
@@ -3989,7 +3935,7 @@ public class IndexWriter implements Clos
}
if (infoStream != null) {
- message("done all syncs");
+ infoStream.message("IW", "done all syncs");
}
assert testPoint("midStartCommitSuccess");
@@ -4004,7 +3950,7 @@ public class IndexWriter implements Clos
if (!pendingCommitSet) {
if (infoStream != null) {
- message("hit exception committing segments file");
+ infoStream.message("IW", "hit exception committing segments file");
}
// Hit exception
@@ -4057,7 +4003,7 @@ public class IndexWriter implements Clos
private void handleOOM(OutOfMemoryError oom, String location) {
if (infoStream != null) {
- message("hit OutOfMemoryError inside " + location);
+ infoStream.message("IW", "hit OutOfMemoryError inside " + location);
}
hitOOM = true;
throw oom;
@@ -4082,7 +4028,7 @@ public class IndexWriter implements Clos
//System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any()));
ensureOpen();
if (infoStream != null) {
- message("nrtIsCurrent: infoVersion matches: " + (infos.version == segmentInfos.version) + " DW changes: " + docWriter.anyChanges() + " BD changes: "+bufferedDeletesStream.any());
+ infoStream.message("IW", "nrtIsCurrent: infoVersion matches: " + (infos.version == segmentInfos.version) + " DW changes: " + docWriter.anyChanges() + " BD changes: "+bufferedDeletesStream.any());
}
return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java Wed Nov 9 14:52:33 2011
@@ -17,12 +17,16 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.PrintStream;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.util.InfoStream;
+import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.Version;
/**
@@ -122,6 +126,7 @@ public final class IndexWriterConfig imp
private volatile IndexingChain indexingChain;
private volatile IndexReaderWarmer mergedSegmentWarmer;
private volatile Codec codec;
+ private volatile InfoStream infoStream;
private volatile MergePolicy mergePolicy;
private volatile DocumentsWriterPerThreadPool indexerThreadPool;
private volatile boolean readerPooling;
@@ -159,6 +164,7 @@ public final class IndexWriterConfig imp
indexingChain = DocumentsWriterPerThread.defaultIndexingChain;
mergedSegmentWarmer = null;
codec = Codec.getDefault();
+ infoStream = InfoStream.getDefault();
if (matchVersion.onOrAfter(Version.LUCENE_32)) {
mergePolicy = new TieredMergePolicy();
} else {
@@ -676,6 +682,30 @@ public final class IndexWriterConfig imp
public FlushPolicy getFlushPolicy() {
return flushPolicy;
}
+
+ /**
+ * @see #setInfoStream(InfoStream)
+ */
+ public InfoStream getInfoStream() {
+ return infoStream;
+ }
+
+ /** If non-null, information about merges, deletes and a
+ * message when maxFieldLength is reached will be printed
+ * to this.
+ */
+ public IndexWriterConfig setInfoStream(InfoStream infoStream) {
+ this.infoStream = infoStream;
+ return this;
+ }
+
+ /**
+ * Convenience method that uses {@link PrintStreamInfoStream}
+ */
+ public IndexWriterConfig setInfoStream(PrintStream printStream) {
+ this.infoStream = printStream == null ? null : new PrintStreamInfoStream(printStream);
+ return this;
+ }
@Override
public String toString() {
@@ -695,6 +725,7 @@ public final class IndexWriterConfig imp
sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n");
sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n");
sb.append("codec=").append(codec).append("\n");
+ sb.append("infoStream=").append(infoStream == null ? "null" : infoStream.getClass().getName()).append("\n");
sb.append("mergePolicy=").append(mergePolicy).append("\n");
sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n");
sb.append("readerPooling=").append(readerPooling).append("\n");
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Wed Nov 9 14:52:33 2011
@@ -23,6 +23,8 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
+import org.apache.lucene.util.InfoStream;
+
/** <p>This class implements a {@link MergePolicy} that tries
* to merge segments into levels of exponentially
* increasing size, where each level has fewer segments than
@@ -104,8 +106,12 @@ public abstract class LogMergePolicy ext
}
protected void message(String message) {
- if (verbose())
- writer.get().message("LMP: " + message);
+ if (verbose()) {
+ final InfoStream infoStream = writer.get().infoStream;
+ if (infoStream != null) {
+ infoStream.message("LMP", message);
+ }
+ }
}
/** <p>Returns the number of segments that are merged at
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java Wed Nov 9 14:52:33 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.codecs.Pe
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.InfoStream;
/**
* Encapsulates all necessary state to initiate a {@link PerDocConsumer} and
@@ -29,7 +30,7 @@ import org.apache.lucene.util.Counter;
* @lucene.experimental
*/
public class PerDocWriteState {
- public final PrintStream infoStream;
+ public final InfoStream infoStream;
public final Directory directory;
public final String segmentName;
public final FieldInfos fieldInfos;
@@ -37,7 +38,7 @@ public class PerDocWriteState {
public final String segmentSuffix;
public final IOContext context;
- public PerDocWriteState(PrintStream infoStream, Directory directory,
+ public PerDocWriteState(InfoStream infoStream, Directory directory,
String segmentName, FieldInfos fieldInfos, Counter bytesUsed,
String segmentSuffix, IOContext context) {
this.infoStream = infoStream;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Wed Nov 9 14:52:33 2011
@@ -23,7 +23,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.FieldsReader;
+import org.apache.lucene.index.codecs.StoredFieldsReader;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
@@ -55,7 +55,7 @@ final class SegmentCoreReaders {
private final SegmentReader owner;
- FieldsReader fieldsReaderOrig;
+ StoredFieldsReader fieldsReaderOrig;
TermVectorsReader termVectorsReaderOrig;
CompoundFileDirectory cfsReader;
CompoundFileDirectory storeCFSReader;
@@ -110,7 +110,7 @@ final class SegmentCoreReaders {
return termVectorsReaderOrig;
}
- synchronized FieldsReader getFieldsReaderOrig() {
+ synchronized StoredFieldsReader getFieldsReaderOrig() {
return fieldsReaderOrig;
}
@@ -166,14 +166,8 @@ final class SegmentCoreReaders {
}
final String storesSegment = si.getDocStoreSegment();
- fieldsReaderOrig = si.getCodec().fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context,
- si.getDocStoreOffset(), si.docCount);
-
- // Verify two sources of "maxDoc" agree:
- if (si.getDocStoreOffset() == -1 && fieldsReaderOrig.size() != si.docCount) {
- throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + fieldsReaderOrig.size() + " but segmentInfo shows " + si.docCount);
- }
-
+ fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(storeDir, si, fieldInfos, context);
+
if (si.getHasVectors()) { // open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Wed Nov 9 14:52:33 2011
@@ -285,6 +285,8 @@ public final class SegmentInfo implement
* store files otherwise.
*/
public long sizeInBytes(boolean includeDocStores) throws IOException {
+ // TODO: based on how this is used, can't we just forget about all this docstore crap?
+ // its really an abstraction violation into the codec
if (includeDocStores) {
if (sizeInBytesWithStore != -1) {
return sizeInBytesWithStore;
@@ -293,7 +295,7 @@ public final class SegmentInfo implement
for (final String fileName : files()) {
// We don't count bytes used by a shared doc store
// against this segment
- if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName)) {
+ if (docStoreOffset == -1 || !isDocStoreFile(fileName)) {
sum += dir.fileLength(fileName);
}
}
@@ -305,7 +307,7 @@ public final class SegmentInfo implement
}
long sum = 0;
for (final String fileName : files()) {
- if (IndexFileNames.isDocStoreFile(fileName)) {
+ if (isDocStoreFile(fileName)) {
continue;
}
sum += dir.fileLength(fileName);
@@ -314,6 +316,13 @@ public final class SegmentInfo implement
return sizeInBytesNoStore;
}
}
+
+ // TODO: a little messy, but sizeInBytes above that uses this is the real problem.
+ private boolean isDocStoreFile(String fileName) throws IOException {
+ Set<String> docStoreFiles = new HashSet<String>();
+ codec.storedFieldsFormat().files(dir, this, docStoreFiles);
+ return IndexFileNames.isDocStoreFile(fileName) || docStoreFiles.contains(fileName);
+ }
public boolean getHasVectors() throws IOException {
return hasVectors == CHECK_FIELDINFO ? getFieldInfos().hasVectors() : hasVectors == YES;
@@ -634,11 +643,10 @@ public final class SegmentInfo implement
// We are sharing doc stores (stored fields, term
// vectors) with other segments
assert docStoreSegment != null;
+ // TODO: push this out into preflex fieldsFormat?
if (docStoreIsCompoundFile) {
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION));
} else {
- fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_INDEX_EXTENSION));
- fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_EXTENSION));
if (getHasVectors()) {
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
@@ -646,8 +654,6 @@ public final class SegmentInfo implement
}
}
} else if (!useCompoundFile) {
- fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.FIELDS_INDEX_EXTENSION));
- fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.FIELDS_EXTENSION));
if (getHasVectors()) {
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Wed Nov 9 14:52:33 2011
@@ -23,23 +23,20 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import org.apache.lucene.document.Document;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.MergePolicy.MergeAbortedException;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
-import org.apache.lucene.index.codecs.FieldsReader;
-import org.apache.lucene.index.codecs.FieldsWriter;
-import org.apache.lucene.index.codecs.MergeState;
+import org.apache.lucene.index.codecs.StoredFieldsWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.ReaderUtil;
/**
@@ -51,52 +48,33 @@ import org.apache.lucene.util.ReaderUtil
* @see #add
*/
final class SegmentMerger {
- private Directory directory;
- private String segment;
- private int termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL;
-
- private List<MergeState.IndexReaderAndLiveDocs> readers = new ArrayList<MergeState.IndexReaderAndLiveDocs>();
- private final FieldInfos fieldInfos;
-
- private int mergedDocs;
-
- private final MergeState.CheckAbort checkAbort;
+ private final Directory directory;
+ private final String segment;
+ private final int termIndexInterval;
/** Maximum number of contiguous documents to bulk-copy
- when merging stored fields */
+ when merging term vectors */
private final static int MAX_RAW_MERGE_DOCS = 4192;
- private Codec codec;
- private SegmentWriteState segmentWriteState;
-
- private PayloadProcessorProvider payloadProcessorProvider;
+ private final Codec codec;
- private IOContext context;
+ private final IOContext context;
+
+ private final MergeState mergeState = new MergeState();
- SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) {
- this.payloadProcessorProvider = payloadProcessorProvider;
+ SegmentMerger(InfoStream infoStream, Directory dir, int termIndexInterval, String name, MergeState.CheckAbort checkAbort, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) {
+ mergeState.infoStream = infoStream;
+ mergeState.readers = new ArrayList<MergeState.IndexReaderAndLiveDocs>();
+ mergeState.fieldInfos = fieldInfos;
+ mergeState.checkAbort = checkAbort;
+ mergeState.payloadProcessorProvider = payloadProcessorProvider;
directory = dir;
segment = name;
- this.fieldInfos = fieldInfos;
- if (merge != null) {
- checkAbort = new MergeState.CheckAbort(merge, directory);
- } else {
- checkAbort = new MergeState.CheckAbort(null, null) {
- @Override
- public void work(double units) throws MergeAbortedException {
- // do nothing
- }
- };
- }
this.termIndexInterval = termIndexInterval;
this.codec = codec;
this.context = context;
}
- public FieldInfos fieldInfos() {
- return fieldInfos;
- }
-
/**
* Add an IndexReader to the collection of readers that are to be merged
* @param reader
@@ -106,7 +84,7 @@ final class SegmentMerger {
new ReaderUtil.Gather(reader) {
@Override
protected void add(int base, IndexReader r) {
- readers.add(new MergeState.IndexReaderAndLiveDocs(r, r.getLiveDocs()));
+ mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(r, r.getLiveDocs()));
}
}.run();
} catch (IOException ioe) {
@@ -116,7 +94,7 @@ final class SegmentMerger {
}
final void add(SegmentReader reader, Bits liveDocs) {
- readers.add(new MergeState.IndexReaderAndLiveDocs(reader, liveDocs));
+ mergeState.readers.add(new MergeState.IndexReaderAndLiveDocs(reader, liveDocs));
}
/**
@@ -125,25 +103,36 @@ final class SegmentMerger {
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- final int merge() throws CorruptIndexException, IOException {
+ final MergeState merge() throws CorruptIndexException, IOException {
// NOTE: it's important to add calls to
// checkAbort.work(...) if you make any changes to this
// method that will spend alot of time. The frequency
// of this check impacts how long
// IndexWriter.close(false) takes to actually stop the
// threads.
+
+ final int numReaders = mergeState.readers.size();
+ // Remap docIDs
+ mergeState.docMaps = new int[numReaders][];
+ mergeState.docBase = new int[numReaders];
+ mergeState.dirPayloadProcessor = new PayloadProcessorProvider.DirPayloadProcessor[numReaders];
+ mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[numReaders];
+
+ mergeFieldInfos();
+ setMatchingSegmentReaders();
+ mergeState.mergedDocCount = mergeFields();
- mergedDocs = mergeFields();
- mergeTerms();
- mergePerDoc();
+ final SegmentWriteState segmentWriteState = new SegmentWriteState(mergeState.infoStream, directory, segment, mergeState.fieldInfos, mergeState.mergedDocCount, termIndexInterval, codec, null, context);
+ mergeTerms(segmentWriteState);
+ mergePerDoc(segmentWriteState);
mergeNorms();
- if (fieldInfos.hasVectors()) {
- mergeVectors();
+ if (mergeState.fieldInfos.hasVectors()) {
+ mergeVectors(segmentWriteState);
}
// write FIS once merge is done. IDV might change types or drops fields
- fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
- return mergedDocs;
+ mergeState.fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
+ return mergeState;
}
/**
@@ -165,7 +154,7 @@ final class SegmentMerger {
assert !IndexFileNames.isSeparateNormsFile(file)
: "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
directory.copy(cfsDir, file, file, context);
- checkAbort.work(directory.fileLength(file));
+ mergeState.checkAbort.work(directory.fileLength(file));
}
} finally {
cfsDir.close();
@@ -186,195 +175,81 @@ final class SegmentMerger {
}
}
- private SegmentReader[] matchingSegmentReaders;
- private int[] rawDocLengths;
- private int[] rawDocLengths2;
- private int matchedCount;
-
- public int getMatchedSubReaderCount() {
- return matchedCount;
- }
-
private void setMatchingSegmentReaders() {
// If the i'th reader is a SegmentReader and has
// identical fieldName -> number mapping, then this
// array will be non-null at position i:
- int numReaders = readers.size();
- matchingSegmentReaders = new SegmentReader[numReaders];
+ int numReaders = mergeState.readers.size();
+ mergeState.matchingSegmentReaders = new SegmentReader[numReaders];
// If this reader is a SegmentReader, and all of its
// field name -> number mappings match the "merged"
// FieldInfos, then we can do a bulk copy of the
// stored fields:
for (int i = 0; i < numReaders; i++) {
- MergeState.IndexReaderAndLiveDocs reader = readers.get(i);
+ MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i);
if (reader.reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader.reader;
boolean same = true;
FieldInfos segmentFieldInfos = segmentReader.fieldInfos();
for (FieldInfo fi : segmentFieldInfos) {
- same = fieldInfos.fieldName(fi.number).equals(fi.name);
+ same = mergeState.fieldInfos.fieldName(fi.number).equals(fi.name);
}
if (same) {
- matchingSegmentReaders[i] = segmentReader;
- matchedCount++;
+ mergeState.matchingSegmentReaders[i] = segmentReader;
+ mergeState.matchedCount++;
}
}
}
- // Used for bulk-reading raw bytes for stored fields
- rawDocLengths = new int[MAX_RAW_MERGE_DOCS];
- rawDocLengths2 = new int[MAX_RAW_MERGE_DOCS];
+ if (mergeState.infoStream != null) {
+ mergeState.infoStream.message("SM", "merge store matchedCount=" + mergeState.matchedCount + " vs " + mergeState.readers.size());
+ if (mergeState.matchedCount != mergeState.readers.size()) {
+ mergeState.infoStream.message("SM", "" + (mergeState.readers.size() - mergeState.matchedCount) + " non-bulk merges");
+ }
+ }
}
- /**
- *
- * @return The number of documents in all of the readers
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- private int mergeFields() throws CorruptIndexException, IOException {
- for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : readers) {
+ private void mergeFieldInfos() throws IOException {
+ for (MergeState.IndexReaderAndLiveDocs readerAndLiveDocs : mergeState.readers) {
final IndexReader reader = readerAndLiveDocs.reader;
if (reader instanceof SegmentReader) {
SegmentReader segmentReader = (SegmentReader) reader;
FieldInfos readerFieldInfos = segmentReader.fieldInfos();
for (FieldInfo fi : readerFieldInfos) {
- fieldInfos.add(fi);
+ mergeState.fieldInfos.add(fi);
}
} else {
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR), true, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.OMIT_POSITIONS), false, false, false, false, IndexOptions.DOCS_AND_FREQS);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, IndexOptions.DOCS_ONLY);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
- fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.UNINDEXED), false);
- fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION), true, true, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_OFFSET), true, false, true, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR), true, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.OMIT_POSITIONS), false, false, false, false, IndexOptions.DOCS_AND_FREQS);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, IndexOptions.DOCS_ONLY);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ addIndexed(reader, mergeState.fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ mergeState.fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.UNINDEXED), false);
+ mergeState.fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false);
}
}
+ }
+ /**
+ *
+ * @return The number of documents in all of the readers
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
+ private int mergeFields() throws CorruptIndexException, IOException {
int docCount = 0;
- setMatchingSegmentReaders();
- final FieldsWriter fieldsWriter = codec.fieldsFormat().fieldsWriter(directory, segment, context);
+ final StoredFieldsWriter fieldsWriter = codec.storedFieldsFormat().fieldsWriter(directory, segment, context);
try {
- int idx = 0;
- for (MergeState.IndexReaderAndLiveDocs reader : readers) {
- final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++];
- FieldsReader matchingFieldsReader = null;
- if (matchingSegmentReader != null) {
- final FieldsReader fieldsReader = matchingSegmentReader.getFieldsReader();
- if (fieldsReader != null) {
- matchingFieldsReader = fieldsReader;
- }
- }
- if (reader.liveDocs != null) {
- docCount += copyFieldsWithDeletions(fieldsWriter,
- reader, matchingFieldsReader);
- } else {
- docCount += copyFieldsNoDeletions(fieldsWriter,
- reader, matchingFieldsReader);
- }
- }
+ docCount = fieldsWriter.merge(mergeState);
} finally {
fieldsWriter.close();
}
- final String fileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION);
- final long fdxFileLength = directory.fileLength(fileName);
-
- if (4+((long) docCount)*8 != fdxFileLength)
- // This is most likely a bug in Sun JRE 1.6.0_04/_05;
- // we detect that the bug has struck, here, and
- // throw an exception to prevent the corruption from
- // entering the index. See LUCENE-1282 for
- // details.
- throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption");
- segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codec, null, context);
-
- return docCount;
- }
-
- private int copyFieldsWithDeletions(final FieldsWriter fieldsWriter, final MergeState.IndexReaderAndLiveDocs reader,
- final FieldsReader matchingFieldsReader)
- throws IOException, MergeAbortedException, CorruptIndexException {
- int docCount = 0;
- final int maxDoc = reader.reader.maxDoc();
- final Bits liveDocs = reader.liveDocs;
- assert liveDocs != null;
- if (matchingFieldsReader != null) {
- // We can bulk-copy because the fieldInfos are "congruent"
- for (int j = 0; j < maxDoc;) {
- if (!liveDocs.get(j)) {
- // skip deleted docs
- ++j;
- continue;
- }
- // We can optimize this case (doing a bulk byte copy) since the field
- // numbers are identical
- int start = j, numDocs = 0;
- do {
- j++;
- numDocs++;
- if (j >= maxDoc) break;
- if (!liveDocs.get(j)) {
- j++;
- break;
- }
- } while(numDocs < MAX_RAW_MERGE_DOCS);
-
- IndexInput stream = matchingFieldsReader.rawDocs(rawDocLengths, start, numDocs);
- fieldsWriter.addRawDocuments(stream, rawDocLengths, numDocs);
- docCount += numDocs;
- checkAbort.work(300 * numDocs);
- }
- } else {
- for (int j = 0; j < maxDoc; j++) {
- if (!liveDocs.get(j)) {
- // skip deleted docs
- continue;
- }
- // TODO: this could be more efficient using
- // FieldVisitor instead of loading/writing entire
- // doc; ie we just have to renumber the field number
- // on the fly?
- // NOTE: it's very important to first assign to doc then pass it to
- // termVectorsWriter.addAllDocVectors; see LUCENE-1282
- Document doc = reader.reader.document(j);
- fieldsWriter.addDocument(doc, fieldInfos);
- docCount++;
- checkAbort.work(300);
- }
- }
- return docCount;
- }
-
- private int copyFieldsNoDeletions(final FieldsWriter fieldsWriter, final MergeState.IndexReaderAndLiveDocs reader,
- final FieldsReader matchingFieldsReader)
- throws IOException, MergeAbortedException, CorruptIndexException {
- final int maxDoc = reader.reader.maxDoc();
- int docCount = 0;
- if (matchingFieldsReader != null) {
- // We can bulk-copy because the fieldInfos are "congruent"
- while (docCount < maxDoc) {
- int len = Math.min(MAX_RAW_MERGE_DOCS, maxDoc - docCount);
- IndexInput stream = matchingFieldsReader.rawDocs(rawDocLengths, docCount, len);
- fieldsWriter.addRawDocuments(stream, rawDocLengths, len);
- docCount += len;
- checkAbort.work(300 * len);
- }
- } else {
- for (; docCount < maxDoc; docCount++) {
- // NOTE: it's very important to first assign to doc then pass it to
- // termVectorsWriter.addAllDocVectors; see LUCENE-1282
- Document doc = reader.reader.document(docCount);
- fieldsWriter.addDocument(doc, fieldInfos);
- checkAbort.work(300);
- }
- }
return docCount;
}
@@ -382,13 +257,15 @@ final class SegmentMerger {
* Merge the TermVectors from each of the segments into the new one.
* @throws IOException
*/
- private final void mergeVectors() throws IOException {
- TermVectorsWriter termVectorsWriter = new TermVectorsWriter(directory, segment, fieldInfos, context);
-
+ private final void mergeVectors(SegmentWriteState segmentWriteState) throws IOException {
+ TermVectorsWriter termVectorsWriter = new TermVectorsWriter(directory, segment, mergeState.fieldInfos, context);
+ // Used for bulk-reading raw bytes for term vectors
+ int rawDocLengths[] = new int[MAX_RAW_MERGE_DOCS];
+ int rawDocLengths2[] = new int[MAX_RAW_MERGE_DOCS];
try {
int idx = 0;
- for (final MergeState.IndexReaderAndLiveDocs reader : readers) {
- final SegmentReader matchingSegmentReader = matchingSegmentReaders[idx++];
+ for (final MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) {
+ final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++];
TermVectorsReader matchingVectorsReader = null;
if (matchingSegmentReader != null) {
TermVectorsReader vectorsReader = matchingSegmentReader.getTermVectorsReader();
@@ -399,9 +276,9 @@ final class SegmentMerger {
}
}
if (reader.liveDocs != null) {
- copyVectorsWithDeletions(termVectorsWriter, matchingVectorsReader, reader);
+ copyVectorsWithDeletions(termVectorsWriter, matchingVectorsReader, reader, rawDocLengths, rawDocLengths2);
} else {
- copyVectorsNoDeletions(termVectorsWriter, matchingVectorsReader, reader);
+ copyVectorsNoDeletions(termVectorsWriter, matchingVectorsReader, reader, rawDocLengths, rawDocLengths2);
}
}
} finally {
@@ -410,6 +287,7 @@ final class SegmentMerger {
final String fileName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION);
final long tvxSize = directory.fileLength(fileName);
+ final int mergedDocs = segmentWriteState.numDocs;
if (4+((long) mergedDocs)*16 != tvxSize)
// This is most likely a bug in Sun JRE 1.6.0_04/_05;
@@ -422,7 +300,9 @@ final class SegmentMerger {
private void copyVectorsWithDeletions(final TermVectorsWriter termVectorsWriter,
final TermVectorsReader matchingVectorsReader,
- final MergeState.IndexReaderAndLiveDocs reader)
+ final MergeState.IndexReaderAndLiveDocs reader,
+ int rawDocLengths[],
+ int rawDocLengths2[])
throws IOException, MergeAbortedException {
final int maxDoc = reader.reader.maxDoc();
final Bits liveDocs = reader.liveDocs;
@@ -449,7 +329,7 @@ final class SegmentMerger {
matchingVectorsReader.rawDocs(rawDocLengths, rawDocLengths2, start, numDocs);
termVectorsWriter.addRawDocuments(matchingVectorsReader, rawDocLengths, rawDocLengths2, numDocs);
- checkAbort.work(300 * numDocs);
+ mergeState.checkAbort.work(300 * numDocs);
}
} else {
for (int docNum = 0; docNum < maxDoc; docNum++) {
@@ -462,14 +342,16 @@ final class SegmentMerger {
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
TermFreqVector[] vectors = reader.reader.getTermFreqVectors(docNum);
termVectorsWriter.addAllDocVectors(vectors);
- checkAbort.work(300);
+ mergeState.checkAbort.work(300);
}
}
}
private void copyVectorsNoDeletions(final TermVectorsWriter termVectorsWriter,
final TermVectorsReader matchingVectorsReader,
- final MergeState.IndexReaderAndLiveDocs reader)
+ final MergeState.IndexReaderAndLiveDocs reader,
+ int rawDocLengths[],
+ int rawDocLengths2[])
throws IOException, MergeAbortedException {
final int maxDoc = reader.reader.maxDoc();
if (matchingVectorsReader != null) {
@@ -480,7 +362,7 @@ final class SegmentMerger {
matchingVectorsReader.rawDocs(rawDocLengths, rawDocLengths2, docCount, len);
termVectorsWriter.addRawDocuments(matchingVectorsReader, rawDocLengths, rawDocLengths2, len);
docCount += len;
- checkAbort.work(300 * len);
+ mergeState.checkAbort.work(300 * len);
}
} else {
for (int docNum = 0; docNum < maxDoc; docNum++) {
@@ -488,27 +370,18 @@ final class SegmentMerger {
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
TermFreqVector[] vectors = reader.reader.getTermFreqVectors(docNum);
termVectorsWriter.addAllDocVectors(vectors);
- checkAbort.work(300);
+ mergeState.checkAbort.work(300);
}
}
}
- Codec getCodec() {
- assert segmentWriteState != null;
- return segmentWriteState.codec;
- }
-
- private final void mergeTerms() throws CorruptIndexException, IOException {
-
- // Let CodecProvider decide which codec will be used to write
- // the new segment:
-
+ private final void mergeTerms(SegmentWriteState segmentWriteState) throws CorruptIndexException, IOException {
int docBase = 0;
final List<Fields> fields = new ArrayList<Fields>();
final List<ReaderUtil.Slice> slices = new ArrayList<ReaderUtil.Slice>();
- for(MergeState.IndexReaderAndLiveDocs r : readers) {
+ for(MergeState.IndexReaderAndLiveDocs r : mergeState.readers) {
final Fields f = r.reader.fields();
final int maxDoc = r.reader.maxDoc();
if (f != null) {
@@ -518,30 +391,15 @@ final class SegmentMerger {
docBase += maxDoc;
}
- // we may gather more readers than mergeState.readerCount
- mergeState = new MergeState();
- mergeState.readers = readers;
- mergeState.readerCount = readers.size();
- mergeState.fieldInfos = fieldInfos;
- mergeState.mergedDocCount = mergedDocs;
-
- // Remap docIDs
- mergeState.docMaps = new int[mergeState.readerCount][];
- mergeState.docBase = new int[mergeState.readerCount];
- mergeState.hasPayloadProcessorProvider = payloadProcessorProvider != null;
- mergeState.dirPayloadProcessor = new PayloadProcessorProvider.DirPayloadProcessor[mergeState.readerCount];
- mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[mergeState.readerCount];
- mergeState.checkAbort = checkAbort;
+ final int numReaders = mergeState.readers.size();
docBase = 0;
- int inputDocBase = 0;
- for(int i=0;i<mergeState.readerCount;i++) {
+ for(int i=0;i<numReaders;i++) {
- final MergeState.IndexReaderAndLiveDocs reader = readers.get(i);
+ final MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i);
mergeState.docBase[i] = docBase;
- inputDocBase += reader.reader.maxDoc();
final int maxDoc = reader.reader.maxDoc();
if (reader.liveDocs != null) {
int delCount = 0;
@@ -562,8 +420,8 @@ final class SegmentMerger {
docBase += maxDoc;
}
- if (payloadProcessorProvider != null) {
- mergeState.dirPayloadProcessor[i] = payloadProcessorProvider.getDirProcessor(reader.reader.directory());
+ if (mergeState.payloadProcessorProvider != null) {
+ mergeState.dirPayloadProcessor[i] = mergeState.payloadProcessorProvider.getDirProcessor(reader.reader.directory());
}
}
@@ -583,7 +441,7 @@ final class SegmentMerger {
}
}
- private void mergePerDoc() throws IOException {
+ private void mergePerDoc(SegmentWriteState segmentWriteState) throws IOException {
final PerDocConsumer docsConsumer = codec.docValuesFormat()
.docsConsumer(new PerDocWriteState(segmentWriteState));
// TODO: remove this check when 3.x indexes are no longer supported
@@ -603,25 +461,18 @@ final class SegmentMerger {
}
}
}
-
- private MergeState mergeState;
-
- public boolean getAnyNonBulkMerges() {
- assert matchedCount <= readers.size();
- return matchedCount != readers.size();
- }
private void mergeNorms() throws IOException {
IndexOutput output = null;
boolean success = false;
try {
- for (FieldInfo fi : fieldInfos) {
+ for (FieldInfo fi : mergeState.fieldInfos) {
if (fi.isIndexed && !fi.omitNorms) {
if (output == null) {
output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION), context);
output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length);
}
- for (MergeState.IndexReaderAndLiveDocs reader : readers) {
+ for (MergeState.IndexReaderAndLiveDocs reader : mergeState.readers) {
final int maxDoc = reader.reader.maxDoc();
byte normBuffer[] = reader.reader.norms(fi.name);
if (normBuffer == null) {
@@ -643,7 +494,7 @@ final class SegmentMerger {
}
}
}
- checkAbort.work(maxDoc);
+ mergeState.checkAbort.work(maxDoc);
}
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentReader.java Wed Nov 9 14:52:33 2011
@@ -30,7 +30,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.codecs.FieldsReader;
+import org.apache.lucene.index.codecs.StoredFieldsReader;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
@@ -47,7 +47,7 @@ public class SegmentReader extends Index
private SegmentInfo si;
private final ReaderContext readerContext = new AtomicReaderContext(this);
- CloseableThreadLocal<FieldsReader> fieldsReaderLocal = new FieldsReaderLocal();
+ CloseableThreadLocal<StoredFieldsReader> fieldsReaderLocal = new FieldsReaderLocal();
CloseableThreadLocal<TermVectorsReader> termVectorsLocal = new CloseableThreadLocal<TermVectorsReader>();
volatile BitVector liveDocs;
@@ -74,9 +74,9 @@ public class SegmentReader extends Index
/**
* Sets the initial value
*/
- private class FieldsReaderLocal extends CloseableThreadLocal<FieldsReader> {
+ private class FieldsReaderLocal extends CloseableThreadLocal<StoredFieldsReader> {
@Override
- protected FieldsReader initialValue() {
+ protected StoredFieldsReader initialValue() {
return core.getFieldsReaderOrig().clone();
}
}
@@ -368,7 +368,8 @@ public class SegmentReader extends Index
hasChanges = false;
}
- FieldsReader getFieldsReader() {
+ /** @lucene.internal */
+ public StoredFieldsReader getFieldsReader() {
return fieldsReaderLocal.get();
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java Wed Nov 9 14:52:33 2011
@@ -17,18 +17,17 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.PrintStream;
-
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.BitVector;
+import org.apache.lucene.util.InfoStream;
/**
* @lucene.experimental
*/
public class SegmentWriteState {
- public final PrintStream infoStream;
+ public final InfoStream infoStream;
public final Directory directory;
public final String segmentName;
public final FieldInfos fieldInfos;
@@ -56,7 +55,7 @@ public class SegmentWriteState {
public final IOContext context;
- public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
+ public SegmentWriteState(InfoStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
int numDocs, int termIndexInterval, Codec codec, BufferedDeletes segDeletes, IOContext context) {
this.infoStream = infoStream;
this.segDeletes = segDeletes;
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/StoredFieldVisitor.java Wed Nov 9 14:52:33 2011
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
-import org.apache.lucene.store.IndexInput;
/**
* Expert: provides a low-level means of accessing the stored field
@@ -35,54 +34,39 @@ import org.apache.lucene.store.IndexInpu
*
* @lucene.experimental */
-public class StoredFieldVisitor {
- /** Process a binary field. Note that if you want to
- * skip the field you must seek the IndexInput
- * (e.g., call <code>in.seek(numUTF8Bytes + in.getFilePointer()</code>)
- *
- * <p>Return true to stop loading fields. */
- public boolean binaryField(FieldInfo fieldInfo, IndexInput in, int numBytes) throws IOException {
- in.seek(in.getFilePointer() + numBytes);
- return false;
- }
-
- /** Process a string field by reading numUTF8Bytes.
- * Note that if you want to skip the field you must
- * seek the IndexInput as if you had read numBytes by
- * (e.g., call <code>in.seek(numUTF8Bytes + in.getFilePointer()</code>)
- *
- * <p>Return true to stop loading fields. */
- public boolean stringField(FieldInfo fieldInfo, IndexInput in, int numUTF8Bytes) throws IOException {
- in.seek(in.getFilePointer() + numUTF8Bytes);
- return false;
- }
-
- /** Process a int numeric field.
- *
- * <p>Return true to stop loading fields. */
- public boolean intField(FieldInfo fieldInfo, int value) throws IOException {
- return false;
- }
-
- /** Process a long numeric field.
- *
- * <p>Return true to stop loading fields. */
- public boolean longField(FieldInfo fieldInfo, long value) throws IOException {
- return false;
- }
-
- /** Process a float numeric field.
- *
- * <p>Return true to stop loading fields. */
- public boolean floatField(FieldInfo fieldInfo, float value) throws IOException {
- return false;
- }
-
- /** Process a double numeric field.
- *
- * <p>Return true to stop loading fields. */
- public boolean doubleField(FieldInfo fieldInfo, double value) throws IOException {
- return false;
+public abstract class StoredFieldVisitor {
+ /** Process a binary field. */
+ public void binaryField(FieldInfo fieldInfo, byte[] value, int offset, int length) throws IOException {
}
-}
+ /** Process a string field */
+ public void stringField(FieldInfo fieldInfo, String value) throws IOException {
+ }
+
+ /** Process a int numeric field. */
+ public void intField(FieldInfo fieldInfo, int value) throws IOException {
+ }
+
+ /** Process a long numeric field. */
+ public void longField(FieldInfo fieldInfo, long value) throws IOException {
+ }
+
+ /** Process a float numeric field. */
+ public void floatField(FieldInfo fieldInfo, float value) throws IOException {
+ }
+
+ /** Process a double numeric field. */
+ public void doubleField(FieldInfo fieldInfo, double value) throws IOException {
+ }
+
+ public abstract Status needsField(FieldInfo fieldInfo) throws IOException;
+
+ public static enum Status {
+ /** yes, i want the field */
+ YES,
+ /** no, i do not */
+ NO,
+ /** stop loading fields for this document entirely */
+ STOP
+ }
+}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/TieredMergePolicy.java Wed Nov 9 14:52:33 2011
@@ -26,6 +26,8 @@ import java.util.Comparator;
import java.util.List;
import java.util.ArrayList;
+import org.apache.lucene.util.InfoStream;
+
/**
* Merges segments of approximately equal size, subject to
* an allowed number of segments per tier. This is similar
@@ -648,7 +650,10 @@ public class TieredMergePolicy extends M
private void message(String message) {
if (verbose()) {
- writer.get().message("TMP: " + message);
+ final InfoStream infoStream = writer.get().infoStream;
+ if (infoStream != null) {
+ infoStream.message("TMP", message);
+ }
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java Wed Nov 9 14:52:33 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.InfoStream;
import java.io.IOException;
import java.util.ArrayList;
@@ -151,8 +152,12 @@ public class UpgradeIndexMergePolicy ext
}
private void message(String message) {
- if (verbose())
- writer.get().message("UPGMP: " + message);
+ if (verbose()) {
+ final InfoStream infoStream = writer.get().infoStream;
+ if (infoStream != null) {
+ infoStream.message("UPGMP", message);
+ }
+ }
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Wed Nov 9 14:52:33 2011
@@ -45,7 +45,7 @@ public abstract class Codec implements N
public void files(Directory dir, SegmentInfo info, Set<String> files) throws IOException {
postingsFormat().files(dir, info, "", files);
- //TODO: not yet fieldsFormat().files(dir, info, files);
+ storedFieldsFormat().files(dir, info, files);
docValuesFormat().files(dir, info, files);
}
@@ -55,9 +55,10 @@ public abstract class Codec implements N
/** Encodes/decodes docvalues */
public abstract DocValuesFormat docValuesFormat();
- /** Encodes/decodes stored fields, term vectors, fieldinfos */
- public abstract FieldsFormat fieldsFormat();
+ /** Encodes/decodes stored fields */
+ public abstract StoredFieldsFormat storedFieldsFormat();
+ /** Encodes/decodes segments file */
public abstract SegmentInfosFormat segmentInfosFormat();
/** looks up a codec by name */
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1199779&r1=1199778&r2=1199779&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Wed Nov 9 14:52:33 2011
@@ -69,7 +69,7 @@ public class DefaultSegmentInfosReader e
}
try {
- DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
+ DefaultStoredFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment());
} finally {
// If we opened the directory, close it
if (dir != directory) dir.close();