You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/01/28 14:23:41 UTC
svn commit: r1237057 - in /lucene/dev/branches/lucene3661/lucene/src:
java/org/apache/lucene/codecs/lucene40/
java/org/apache/lucene/codecs/simpletext/ java/org/apache/lucene/index/
java/org/apache/lucene/util/ test/org/apache/lucene/index/
Author: mikemccand
Date: Sat Jan 28 13:23:40 2012
New Revision: 1237057
URL: http://svn.apache.org/viewvc?rev=1237057&view=rev
Log:
LUCENE-3661: remove MutableBits.count()
Added:
lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/index/TestMixedCodecs.java (with props)
Modified:
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/util/MutableBits.java
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java Sat Jan 28 13:23:40 2012
@@ -26,14 +26,20 @@ public class Lucene40LiveDocsFormat exte
@Override
public Bits readLiveDocs(Directory dir, SegmentInfo info, IOContext context) throws IOException {
String filename = IndexFileNames.fileNameFromGeneration(info.name, DELETES_EXTENSION, info.getDelGen());
- return new BitVector(dir, filename, context);
+ final BitVector liveDocs = new BitVector(dir, filename, context);
+ assert liveDocs.count() == info.docCount - info.getDelCount();
+ assert liveDocs.length() == info.docCount;
+ return liveDocs;
}
@Override
public void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfo info, IOContext context) throws IOException {
// nocommit: this api is ugly...
String filename = IndexFileNames.fileNameFromGeneration(info.name, DELETES_EXTENSION, info.getDelGen());
- ((BitVector)bits).write(dir, filename, context);
+ final BitVector liveDocs = (BitVector) bits;
+ assert liveDocs.count() == info.docCount - info.getDelCount();
+ assert liveDocs.length() == info.docCount;
+ liveDocs.write(dir, filename, context);
}
@Override
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java Sat Jan 28 13:23:40 2012
@@ -169,11 +169,6 @@ public class SimpleTextLiveDocsFormat ex
}
@Override
- public int count() {
- return bits.cardinality();
- }
-
- @Override
public SimpleTextBits clone() {
BitSet clonedBits = (BitSet) bits.clone();
return new SimpleTextBits(clonedBits, size);
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Sat Jan 28 13:23:40 2012
@@ -477,7 +477,7 @@ public class DocumentsWriterPerThread {
pendingDeletes.terms.clear();
final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly());
if (infoStream.isEnabled("DWPT")) {
- infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs");
+ infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.delCountOnFlush)) + " deleted docs");
infoStream.message("DWPT", "new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
infoStream.message("DWPT", "flushedFiles=" + newSegment.files());
infoStream.message("DWPT", "flushed codec=" + newSegment.getCodec());
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/IndexWriter.java Sat Jan 28 13:23:40 2012
@@ -468,17 +468,6 @@ public class IndexWriter implements Clos
return reader != null || mergeReader != null;
}
- // Called only from assert
- private boolean countsMatch() {
- if (liveDocs == null) {
- assert pendingDeleteCount == 0;
- } else {
- assert liveDocs.count() == info.docCount - info.getDelCount() - pendingDeleteCount :
- "liveDocs.count()=" + liveDocs.count() + " info.docCount=" + info.docCount + " info.delCount=" + info.getDelCount() + " pendingDelCount=" + pendingDeleteCount;
- }
- return true;
- }
-
// Get reader for searching/deleting
public synchronized SegmentReader getReader(IOContext context) throws IOException {
//System.out.println(" livedocs=" + rld.liveDocs);
@@ -559,7 +548,6 @@ public class IndexWriter implements Clos
getReader(context).decRef();
assert reader != null;
}
- assert countsMatch();
shared = true;
if (liveDocs != null) {
return new SegmentReader(reader, liveDocs, info.docCount - info.getDelCount() - pendingDeleteCount);
@@ -593,7 +581,6 @@ public class IndexWriter implements Clos
//System.out.println("getROLiveDocs seg=" + info);
assert Thread.holdsLock(IndexWriter.this);
shared = true;
- assert countsMatch();
//if (liveDocs != null) {
//System.out.println(" liveCount=" + liveDocs.count());
//}
@@ -612,6 +599,7 @@ public class IndexWriter implements Clos
// Save in case we need to rollback on failure:
final SegmentInfo sav = (SegmentInfo) info.clone();
info.advanceDelGen();
+ info.setDelCount(info.getDelCount() + pendingDeleteCount);
// We can write directly to the actual name (vs to a
// .tmp & renaming it) because the file is not live
@@ -625,9 +613,6 @@ public class IndexWriter implements Clos
info.reset(sav);
}
}
- assert (info.docCount - liveDocs.count()) == info.getDelCount() + pendingDeleteCount:
- "delete count mismatch during commit: seg=" + info + " info.delCount=" + info.getDelCount() + " vs MutableBits=" + (info.docCount-liveDocs.count() + " pendingDelCount=" + pendingDeleteCount);
- info.setDelCount(info.getDelCount() + pendingDeleteCount);
pendingDeleteCount = 0;
return true;
} else {
@@ -3046,7 +3031,6 @@ public class IndexWriter implements Clos
// This means this segment received new deletes
// since we started the merge, so we
// must merge them:
- final int startDocUpto = docUpto;
for(int j=0;j<docCount;j++) {
if (!prevLiveDocs.get(j)) {
assert !currentLiveDocs.get(j);
@@ -3062,8 +3046,6 @@ public class IndexWriter implements Clos
}
}
} else {
- assert mergeState.readers != null;
- assert mergeState.segmentDocCounts != null;
docUpto += mergeState.segmentDocCounts.get(info);
}
} else if (currentLiveDocs != null) {
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Sat Jan 28 13:23:40 2012
@@ -308,6 +308,8 @@ final class SegmentMerger {
final MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i);
+ // nocommit -- assert that final doc count ==
+ // mergedDocCount from stored fields and term vectors
mergeState.docBase[i] = docBase;
final int maxDoc = reader.reader.maxDoc();
final int docCount;
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/index/SegmentReader.java Sat Jan 28 13:23:40 2012
@@ -62,7 +62,6 @@ public final class SegmentReader extends
liveDocs = null;
}
numDocs = si.docCount - si.getDelCount();
- assert checkLiveCounts(false);
success = true;
} finally {
// With lock-less commits, it's entirely possible (and
@@ -94,7 +93,6 @@ public final class SegmentReader extends
// ... but load our own deleted docs:
liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(si.dir, si, context);
numDocs = si.docCount - si.getDelCount();
- assert checkLiveCounts(false);
// We share core w/ parent:
parent.core.incRef();
@@ -114,8 +112,6 @@ public final class SegmentReader extends
this.liveDocs = liveDocs;
this.numDocs = numDocs;
-
- assert checkLiveCounts(true);
}
@Override
@@ -124,26 +120,6 @@ public final class SegmentReader extends
return liveDocs;
}
- private boolean checkLiveCounts(boolean isNRT) throws IOException {
- MutableBits liveDocs = (MutableBits) this.liveDocs;
- if (liveDocs != null) {
- if (liveDocs.length() != si.docCount) {
- throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.length() + " vs segment doc count " + si.docCount + " segment=" + si.name);
- }
-
- final int count = liveDocs.count();
-
- // Verify our docCount matches:
- assert numDocs == count :
- "delete count mismatch: numDocs=" + numDocs + " vs MutableBits=" + (si.docCount-count);
-
- assert isNRT || si.docCount - si.getDelCount() == count :
- "si.docCount=" + si.docCount + "si.getDelCount()=" + si.getDelCount() + " recomputedCount=" + count;
- }
-
- return true;
- }
-
@Override
protected void doClose() throws IOException {
//System.out.println("SR.close seg=" + si);
Modified: lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/util/MutableBits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/util/MutableBits.java?rev=1237057&r1=1237056&r2=1237057&view=diff
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/util/MutableBits.java (original)
+++ lucene/dev/branches/lucene3661/lucene/src/java/org/apache/lucene/util/MutableBits.java Sat Jan 28 13:23:40 2012
@@ -17,10 +17,7 @@ package org.apache.lucene.util;
* limitations under the License.
*/
-public interface MutableBits extends Bits,Cloneable {
+public interface MutableBits extends Bits, Cloneable {
public void clear(int bit);
- // nocommit: remove this from this interface somehow? (used by DWPT infostream at least)
- public int count();
-
public MutableBits clone();
}
Added: lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/index/TestMixedCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/index/TestMixedCodecs.java?rev=1237057&view=auto
==============================================================================
--- lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/index/TestMixedCodecs.java (added)
+++ lucene/dev/branches/lucene3661/lucene/src/test/org/apache/lucene/index/TestMixedCodecs.java Sat Jan 28 13:23:40 2012
@@ -0,0 +1,87 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.preflexrw.PreFlexRWCodec;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
+import org.junit.Assume;
+
+public class TestMixedCodecs extends LuceneTestCase {
+
+ public void test() throws Exception {
+
+ Assume.assumeTrue(!(Codec.getDefault() instanceof PreFlexRWCodec));
+
+ final int NUM_DOCS = atLeast(1000);
+
+ final Directory dir = newDirectory();
+ RandomIndexWriter w = null;
+
+ int docsLeftInThisSegment = 0;
+
+ int docUpto = 0;
+ while (docUpto < NUM_DOCS) {
+ if (docsLeftInThisSegment == 0) {
+ final IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ if (random.nextBoolean()) {
+ // Make sure we aggressively mix in SimpleText
+ // since it has different impls for all codec
+ // formats...
+ iwc.setCodec(Codec.forName("SimpleText"));
+ }
+ if (w != null) {
+ w.close();
+ }
+ w = new RandomIndexWriter(random, dir, iwc);
+ docsLeftInThisSegment = _TestUtil.nextInt(random, 10, 100);
+ }
+ final Document doc = new Document();
+ doc.add(newField("id", String.valueOf(docUpto), StringField.TYPE_STORED));
+ w.addDocument(doc);
+ docUpto++;
+ docsLeftInThisSegment--;
+ }
+
+ // Random delete half the docs:
+ final Set<Integer> deleted = new HashSet<Integer>();
+ while(deleted.size() < NUM_DOCS/2) {
+ final Integer toDelete = random.nextInt(NUM_DOCS);
+ if (!deleted.contains(toDelete)) {
+ deleted.add(toDelete);
+ w.deleteDocuments(new Term("id", String.valueOf(toDelete)));
+ if (random.nextInt(17) == 6) {
+ final IndexReader r = w.getReader();
+ assertEquals(NUM_DOCS - deleted.size(), r.numDocs());
+ r.close();
+ }
+ }
+ }
+
+ w.close();
+ dir.close();
+ }
+}