You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/04/14 12:33:01 UTC
svn commit: r1092105 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/sep/
test/org/apache/lucene/index/
Author: mikemccand
Date: Thu Apr 14 10:33:01 2011
New Revision: 1092105
URL: http://svn.apache.org/viewvc?rev=1092105&view=rev
Log:
LUCENE-3027: disregard FieldInfo.storesPayloads if omitTFAP is true
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1092105&r1=1092104&r2=1092105&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Thu Apr 14 10:33:01 2011
@@ -173,9 +173,10 @@ final class DocFieldProcessorPerThread e
if (totalFieldCount >= fieldHash.length/2)
rehash();
} else {
- fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(),
- field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
- field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ FieldInfo fi = fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(),
+ field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
+ field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ assert !fi.omitTermFreqAndPositions || !fi.storePayloads;
}
if (thisFieldGen != fp.lastGen) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1092105&r1=1092104&r2=1092105&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java Thu Apr 14 10:33:01 2011
@@ -58,6 +58,7 @@ public final class FieldInfo {
this.omitNorms = false;
this.omitTermFreqAndPositions = false;
}
+ assert !omitTermFreqAndPositions || !storePayloads;
}
void setCodecId(int codecId) {
@@ -80,6 +81,7 @@ public final class FieldInfo {
// should only be called by FieldInfos#addOrUpdate
void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector,
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) {
+
if (this.isIndexed != isIndexed) {
this.isIndexed = true; // once indexed, always index
}
@@ -101,7 +103,9 @@ public final class FieldInfo {
}
if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) {
this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life
+ this.storePayloads = false;
}
}
+ assert !this.omitTermFreqAndPositions || !this.storePayloads;
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1092105&r1=1092104&r2=1092105&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java Thu Apr 14 10:33:01 2011
@@ -424,8 +424,8 @@ public final class FieldInfos implements
}
synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
- boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
- boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) {
+ boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
+ boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) {
if (globalFieldNumbers == null) {
throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos");
}
@@ -567,6 +567,7 @@ public final class FieldInfos implements
output.writeVInt(FORMAT_CURRENT);
output.writeVInt(size());
for (FieldInfo fi : this) {
+ assert !fi.omitTermFreqAndPositions || !fi.storePayloads;
byte bits = 0x0;
if (fi.isIndexed) bits |= IS_INDEXED;
if (fi.storeTermVector) bits |= STORE_TERMVECTOR;
@@ -607,6 +608,14 @@ public final class FieldInfos implements
boolean omitNorms = (bits & OMIT_NORMS) != 0;
boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
boolean omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
+
+ // LUCENE-3027: past indices were able to write
+ // storePayloads=true when omitTFAP is also true,
+ // which is invalid. We correct that, here:
+ if (omitTermFreqAndPositions) {
+ storePayloads = false;
+ }
+
final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
addInternal.setCodecId(codecId);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1092105&r1=1092104&r2=1092105&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Thu Apr 14 10:33:01 2011
@@ -104,7 +104,9 @@ final class FreqProxTermsWriter extends
// Aggregate the storePayload as seen by the same
// field across multiple threads
- fieldInfo.storePayloads |= fields[i-start].hasPayloads;
+ if (!fieldInfo.omitTermFreqAndPositions) {
+ fieldInfo.storePayloads |= fields[i-start].hasPayloads;
+ }
}
// If this field has postings then add them to the
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java?rev=1092105&r1=1092104&r2=1092105&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java Thu Apr 14 10:33:01 2011
@@ -177,6 +177,7 @@ class SepSkipListReader extends MultiLev
@Override
protected int readSkipData(int level, IndexInput skipStream) throws IOException {
int delta;
+ assert !omitTF || !currentFieldStoresPayloads;
if (currentFieldStoresPayloads) {
// the current field stores payloads.
// if the doc delta is odd then we have
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1092105&r1=1092104&r2=1092105&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Thu Apr 14 10:33:01 2011
@@ -119,6 +119,7 @@ public class TestOmitTf extends LuceneTe
setMaxBufferedDocs(3).
setMergePolicy(newLogMergePolicy(2))
);
+ writer.setInfoStream(VERBOSE ? System.out : null);
Document d = new Document();
// this field will have Tf