You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/06 14:21:49 UTC

svn commit: r1592731 - in /lucene/dev/trunk/lucene: CHANGES.txt core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java

Author: rmuir
Date: Tue May  6 12:21:49 2014
New Revision: 1592731

URL: http://svn.apache.org/r1592731
Log:
LUCENE-5646: Remove rare/undertested bulk merge algorithm in CompressingStoredFieldsWriter

Modified:
    lucene/dev/trunk/lucene/CHANGES.txt
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
    lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java

Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1592731&r1=1592730&r2=1592731&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue May  6 12:21:49 2014
@@ -84,6 +84,9 @@ Changes in Runtime Behavior
   IndexWriter will now throw IllegalArgumentException.  (Robert Muir,
   Mike McCandless)
 
+* LUCENE-5646: Remove rare/undertested bulk merge algorithm in 
+  CompressingStoredFieldsWriter. (Robert Muir, Adrien Grand)
+
 New Features
 
 * LUCENE-5610: Add Terms.getMin and Terms.getMax to get the lowest and

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java?rev=1592731&r1=1592730&r2=1592731&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsReader.java Tue May  6 12:21:49 2014
@@ -506,17 +506,6 @@ public final class CompressingStoredFiel
     }
 
     /**
-     * Copy compressed data.
-     */
-    void copyCompressedData(DataOutput out) throws IOException {
-      assert getVersion() == VERSION_CURRENT;
-      final long chunkEnd = docBase + chunkDocs == numDocs
-          ? maxPointer
-          : indexReader.getStartPointer(docBase + chunkDocs);
-      out.copyBytes(fieldsStream, chunkEnd - fieldsStream.getFilePointer());
-    }
-
-    /**
      * Check integrity of the data. The iterator is not usable after this method has been called.
      */
     void checkIntegrity() throws IOException {

Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1592731&r1=1592730&r2=1592731&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Tue May  6 12:21:49 2014
@@ -374,36 +374,20 @@ public final class CompressingStoredFiel
               startOffsets[i] = startOffsets[i - 1] + it.lengths[i - 1];
             }
 
-            if (numBufferedDocs == 0 // starting a new chunk
-                && startOffsets[it.chunkDocs - 1] < chunkSize // chunk is small enough
-                && startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] >= chunkSize // chunk is large enough
-                && nextDeletedDoc(it.docBase, liveDocs, it.docBase + it.chunkDocs) == it.docBase + it.chunkDocs) { // no deletion in the chunk
-              assert docID == it.docBase;
-
-              // no need to decompress, just copy data
-              indexWriter.writeIndex(it.chunkDocs, fieldsStream.getFilePointer());
-              writeHeader(this.docBase, it.chunkDocs, it.numStoredFields, it.lengths);
-              it.copyCompressedData(fieldsStream);
-              this.docBase += it.chunkDocs;
-              docID = nextLiveDoc(it.docBase + it.chunkDocs, liveDocs, maxDoc);
-              docCount += it.chunkDocs;
-              mergeState.checkAbort.work(300 * it.chunkDocs);
-            } else {
-              // decompress
-              it.decompress();
-              if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.length) {
-                throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.length);
-              }
-              // copy non-deleted docs
-              for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
-                final int diff = docID - it.docBase;
-                startDocument();
-                bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
-                numStoredFieldsInDoc = it.numStoredFields[diff];
-                finishDocument();
-                ++docCount;
-                mergeState.checkAbort.work(300);
-              }
+            // decompress
+            it.decompress();
+            if (startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] != it.bytes.length) {
+              throw new CorruptIndexException("Corrupted: expected chunk size=" + startOffsets[it.chunkDocs - 1] + it.lengths[it.chunkDocs - 1] + ", got " + it.bytes.length);
+            }
+            // copy non-deleted docs
+            for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
+              final int diff = docID - it.docBase;
+              startDocument();
+              bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+              numStoredFieldsInDoc = it.numStoredFields[diff];
+              finishDocument();
+              ++docCount;
+              mergeState.checkAbort.work(300);
             }
           } while (docID < maxDoc);
 
@@ -424,15 +408,4 @@ public final class CompressingStoredFiel
     }
     return doc;
   }
-
-  private static int nextDeletedDoc(int doc, Bits liveDocs, int maxDoc) {
-    if (liveDocs == null) {
-      return maxDoc;
-    }
-    while (doc < maxDoc && liveDocs.get(doc)) {
-      ++doc;
-    }
-    return doc;
-  }
-
 }