You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/04/12 10:47:48 UTC
svn commit: r1467210 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/compressing/
Author: jpountz
Date: Fri Apr 12 08:47:47 2013
New Revision: 1467210
URL: http://svn.apache.org/r1467210
Log:
LUCENE-4928: Stored fields/Term vectors: put a hard limit on the maximum number of documents per chunk (merged from r1467208).
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1467210&r1=1467209&r2=1467210&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Apr 12 08:47:47 2013
@@ -248,6 +248,13 @@ Bug Fixes
now succeed even if the directory contains a corrupted index (Billow
Gao, Robert Muir, Mike McCandless)
+* LUCENE-4928: Stored fields and term vectors could become super slow in case
+ of tiny documents (a few bytes). This is especially problematic when switching
+ codecs since bulk-merge strategies can't be applied and the same chunk of
+ documents can end up being decompressed thousands of times. A hard limit on
+ the number of documents per chunk has been added to fix this issue.
+ (Robert Muir, Adrien Grand)
+
Documentation
* LUCENE-4841: Added example SimpleSortedSetFacetsExample to show how
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1467210&r1=1467209&r2=1467210&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Fri Apr 12 08:47:47 2013
@@ -53,6 +53,9 @@ import org.apache.lucene.util.packed.Pac
*/
public final class CompressingStoredFieldsWriter extends StoredFieldsWriter {
+ // hard limit on the maximum number of documents per chunk
+ static final int MAX_DOCUMENTS_PER_CHUNK = 128;
+
static final int STRING = 0x00;
static final int BYTE_ARR = 0x01;
static final int NUMERIC_INT = 0x02;
@@ -200,7 +203,7 @@ public final class CompressingStoredFiel
private boolean triggerFlush() {
return bufferedDocs.length >= chunkSize || // chunks of at least chunkSize bytes
- numBufferedDocs >= chunkSize; // can be necessary if most docs are empty
+ numBufferedDocs >= MAX_DOCUMENTS_PER_CHUNK;
}
private void flush() throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java?rev=1467210&r1=1467209&r2=1467210&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsWriter.java Fri Apr 12 08:47:47 2013
@@ -56,6 +56,9 @@ import org.apache.lucene.util.packed.Pac
*/
public final class CompressingTermVectorsWriter extends TermVectorsWriter {
+ // hard limit on the maximum number of documents per chunk
+ static final int MAX_DOCUMENTS_PER_CHUNK = 128;
+
static final String VECTORS_EXTENSION = "tvd";
static final String VECTORS_INDEX_EXTENSION = "tvx";
@@ -322,7 +325,8 @@ public final class CompressingTermVector
}
private boolean triggerFlush() {
- return termSuffixes.length >= chunkSize || pendingDocs.size() >= chunkSize;
+ return termSuffixes.length >= chunkSize
+ || pendingDocs.size() >= MAX_DOCUMENTS_PER_CHUNK;
}
private void flush() throws IOException {