You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/01/28 18:59:45 UTC
svn commit: r1237114 - in
/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index:
IndexWriter.java MergeState.java SegmentMerger.java
Author: mikemccand
Date: Sat Jan 28 17:59:45 2012
New Revision: 1237114
URL: http://svn.apache.org/viewvc?rev=1237114&view=rev
Log:
LUCENE-3661: simplify del count tracking during merge
Modified:
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/MergeState.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1237114&r1=1237113&r2=1237114&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java Sat Jan 28 17:59:45 2012
@@ -447,6 +447,24 @@ public class IndexWriter implements Clos
return rc > myRefCounts;
}
+ // Call only from assert!
+ public synchronized boolean verifyDocCounts() {
+ int count;
+ if (liveDocs != null) {
+ count = 0;
+ for(int docID=0;docID<info.docCount;docID++) {
+ if (liveDocs.get(docID)) {
+ count++;
+ }
+ }
+ } else {
+ count = info.docCount;
+ }
+
+ assert info.docCount - info.getDelCount() - pendingDeleteCount == count: "info.docCount=" + info.docCount + " info.getDelCount()=" + info.getDelCount() + " pendingDeleteCount=" + pendingDeleteCount + " count=" + count;;
+ return true;
+ }
+
// Returns true if any reader remains
public synchronized boolean removeReader(SegmentReader sr, boolean drop) throws IOException {
if (sr == reader) {
@@ -517,6 +535,7 @@ public class IndexWriter implements Clos
public synchronized boolean delete(int docID) {
assert liveDocs != null;
assert docID >= 0 && docID < liveDocs.length();
+ assert !shared;
final boolean didDelete = liveDocs.get(docID);
if (didDelete) {
liveDocs.clear(docID);
@@ -2981,7 +3000,7 @@ public class IndexWriter implements Clos
* saves the resulting deletes file (incrementing the
* delete generation for merge.info). If no deletes were
* flushed, no new deletes file is saved. */
- synchronized private ReadersAndLiveDocs commitMergedDeletes(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
+ synchronized private ReadersAndLiveDocs commitMergedDeletes(MergePolicy.OneMerge merge) throws IOException {
assert testPoint("startCommitMergeDeletes");
@@ -3048,7 +3067,7 @@ public class IndexWriter implements Clos
}
}
} else {
- docUpto += mergeState.segmentDocCounts.get(info);
+ docUpto += info.docCount - info.getDelCount() - rld.pendingDeleteCount;
}
} else if (currentLiveDocs != null) {
// This segment had no deletes before but now it
@@ -3089,7 +3108,7 @@ public class IndexWriter implements Clos
return mergedDeletes;
}
- synchronized private boolean commitMerge(MergePolicy.OneMerge merge, MergeState mergeState) throws IOException {
+ synchronized private boolean commitMerge(MergePolicy.OneMerge merge) throws IOException {
assert testPoint("startCommitMerge");
@@ -3116,7 +3135,7 @@ public class IndexWriter implements Clos
return false;
}
- final ReadersAndLiveDocs mergedDeletes = commitMergedDeletes(merge, mergeState);
+ final ReadersAndLiveDocs mergedDeletes = commitMergedDeletes(merge);
assert mergedDeletes == null || mergedDeletes.pendingDeleteCount != 0;
@@ -3577,6 +3596,8 @@ public class IndexWriter implements Clos
// we pull a copy:
liveDocs = rld.getReadOnlyLiveDocs();
+ assert rld.verifyDocCounts();
+
if (infoStream.isEnabled("IW")) {
if (rld.pendingDeleteCount != 0) {
infoStream.message("IW", "seg=" + info + " delCount=" + info.getDelCount() + " pendingDelCount=" + rld.pendingDeleteCount);
@@ -3589,7 +3610,11 @@ public class IndexWriter implements Clos
}
merge.readerLiveDocs.add(liveDocs);
merge.readers.add(reader);
- merger.add(reader, liveDocs);
+ final int delCount = rld.pendingDeleteCount + info.getDelCount();
+ assert delCount <= info.docCount;
+ if (delCount < info.docCount) {
+ merger.add(reader, liveDocs);
+ }
segUpto++;
}
@@ -3691,7 +3716,7 @@ public class IndexWriter implements Clos
// Force READ context because we merge deletes onto
// this reader:
- if (!commitMerge(merge, mergeState)) {
+ if (!commitMerge(merge)) {
// commitMerge will return false if this merge was aborted
return 0;
}
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/MergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/MergeState.java?rev=1237114&r1=1237113&r2=1237114&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/MergeState.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/MergeState.java Sat Jan 28 17:59:45 2012
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.util.List;
-import java.util.Map;
import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor;
import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor;
@@ -45,7 +44,6 @@ public class MergeState {
public List<IndexReaderAndLiveDocs> readers; // Readers & liveDocs being merged
public int[][] docMaps; // Maps docIDs around deletions
public int[] docBase; // New docID base per reader
- public Map<SegmentInfo,Integer> segmentDocCounts; // Non-deleted docCount per reader
public int mergedDocCount; // Total # merged docs
public CheckAbort checkAbort;
public InfoStream infoStream;
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1237114&r1=1237113&r2=1237114&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Sat Jan 28 17:59:45 2012
@@ -279,18 +279,6 @@ final class SegmentMerger {
}
}
- private int[] shrink(int[] in, int size) {
- final int[] newArray = new int[size];
- System.arraycopy(in, 0, newArray, 0, size);
- return newArray;
- }
-
- private int[][] shrink(int[][] in, int size) {
- final int[][] newArray = new int[size][];
- System.arraycopy(in, 0, newArray, 0, size);
- return newArray;
- }
-
// NOTE: removes any "all deleted" readers from mergeState.readers
private int setDocMaps() throws IOException {
final int numReaders = mergeState.readers.size();
@@ -298,7 +286,6 @@ final class SegmentMerger {
// Remap docIDs
mergeState.docMaps = new int[numReaders][];
mergeState.docBase = new int[numReaders];
- mergeState.segmentDocCounts = new HashMap<SegmentInfo,Integer>();
mergeState.dirPayloadProcessor = new PayloadProcessorProvider.DirPayloadProcessor[numReaders];
mergeState.currentPayloadProcessor = new PayloadProcessorProvider.PayloadProcessor[numReaders];
@@ -332,16 +319,6 @@ final class SegmentMerger {
docMap = null;
}
- if (reader.reader instanceof SegmentReader) {
- mergeState.segmentDocCounts.put(((SegmentReader) reader.reader).getSegmentInfo(), docCount);
- }
-
- if (docCount == 0) {
- // Skip this reader (all docs are deleted):
- mergeState.readers.remove(i);
- continue;
- }
-
mergeState.docMaps[i] = docMap;
docBase += docCount;
@@ -352,13 +329,6 @@ final class SegmentMerger {
i++;
}
- final int numReadersLeft = mergeState.readers.size();
-
- if (numReadersLeft < mergeState.docMaps.length) {
- mergeState.docMaps = shrink(mergeState.docMaps, numReadersLeft);
- mergeState.docBase = shrink(mergeState.docBase, numReadersLeft);
- }
-
return docBase;
}