You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/06 14:21:49 UTC
svn commit: r1592731 - in /lucene/dev/trunk/lucene: CHANGES.txt
core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
Author: rmuir
Date: Tue May 6 12:21:49 2014
New Revision: 1592731
URL: http://svn.apache.org/r1592731
Log:
LUCENE-5646: Remove rare/undertested bulk merge algorithm in CompressingStoredFieldsWriter
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1592731&r1=1592730&r2=1592731&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue May 6 12:21:49 2014
@@ -84,6 +84,9 @@ Changes in Runtime Behavior
IndexWriter will now throw IllegalArgumentException. (Robert Muir,
Mike McCandless)
+* LUCENE-5646: Remove rare/undertested bulk merge algorithm in
+ CompressingStoredFieldsWriter. (Robert Muir, Adrien Grand)
+
New Features
* LUCENE-5610: Add Terms.getMin and Terms.getMax to get the lowest and
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1592731&r1=1592730&r2=1592731&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Tue May 6 12:21:49 2014
@@ -506,17 +506,6 @@ public final class CompressingStoredFiel
}
/**
- * Copy compressed data.
- */
- void copyCompressedData(DataOutput out) throws IOException {
- assert getVersion() == VERSION_CURRENT;
- final long chunkEnd = docBase + chunkDocs == numDocs
- ? maxPointer
- : indexReader.getStartPointer(docBase + chunkDocs);
- out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
- }
-
- /**
* Check integrity of the data. The iterator is not usable after this method has been called.
*/
void checkIntegrity() throws IOException {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1592731&r1=1592730&r2=1592731&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Tue May 6 12:21:49 2014
@@ -374,36 +374,20 @@ public final class CompressingStoredFiel
startOffsets[i] = startOffsets[i - 1] + it.lengths[i - 1];
}
- if (numBufferedDocs == 0 // starting a new chunk
- && startOffsets[it.chunkDocs - 1] < chunkSize // chunk is small enough
- && startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize // chunk is large enough
- && nextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) { // no deletion in the chunk
- assert docID == it.docBase;
-
- // no need to decompress, just copy data
- indexWriter.writeIndex(it.chunkDocs, fieldsStream.getFilePointer());
- writeHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
- it.copyCompressedData(fieldsStream);
- this.docBase += it.chunkDocs;
- docID = nextLiveDoc(it.docBase + it.chunkDocs, liveDocs, maxDoc);
- docCount += it.chunkDocs;
- mergeState.checkAbort.work(300 * it.chunkDocs);
- } else {
- // decompress
- it.decompress();
- if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.length) {
- throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.length);
- }
- // copy non-deleted docs
- for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
- final int diff = docID - it.docBase;
- startDocument();
- bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
- numStoredFieldsInDoc = it.numStoredFields[diff];
- finishDocument();
- ++docCount;
- mergeState.checkAbort.work(300);
- }
+ // decompress
+ it.decompress();
+ if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.length) {
+ throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.length);
+ }
+ // copy non-deleted docs
+ for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
+ final int diff = docID - it.docBase;
+ startDocument();
+ bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+ numStoredFieldsInDoc = it.numStoredFields[diff];
+ finishDocument();
+ ++docCount;
+ mergeState.checkAbort.work(300);
}
} while (docID < maxDoc);
@@ -424,15 +408,4 @@ public final class CompressingStoredFiel
}
return doc;
}
-
- private static int nextDeletedDoc(int doc, Bits liveDocs, int maxDoc) {
- if (liveDocs == null) {
- return maxDoc;
- }
- while (doc < maxDoc && liveDocs.get(doc)) {
- ++doc;
- }
- return doc;
- }
-
}