You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/04/14 12:55:28 UTC
svn commit: r1092118 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/test-framework/ lucene/backwards/src/test/
lucene/contrib/benchmark/ lucene/src/java/org/apache/lucene/index/
lucene/src/test/org/apache/lucene/index/ solr/
Author: mikemccand
Date: Thu Apr 14 10:55:27 2011
New Revision: 1092118
URL: http://svn.apache.org/viewvc?rev=1092118&view=rev
Log:
LUCENE-3027: disregard FieldInfo.storesPayloads if omitTFAP is true
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/lucene/contrib/benchmark/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=1092118&r1=1092117&r2=1092118&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Thu Apr 14 10:55:27 2011
@@ -203,10 +203,11 @@ final class DocFieldProcessorPerThread e
if (totalFieldCount >= fieldHash.length/2)
rehash();
- } else
+ } else {
fp.fieldInfo.update(field.isIndexed(), field.isTermVectorStored(),
field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(),
field.getOmitNorms(), false, field.getOmitTermFreqAndPositions());
+ }
if (thisFieldGen != fp.lastGen) {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1092118&r1=1092117&r2=1092118&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfo.java Thu Apr 14 10:55:27 2011
@@ -54,6 +54,7 @@ final class FieldInfo {
this.omitNorms = true;
this.omitTermFreqAndPositions = false;
}
+ assert !omitTermFreqAndPositions || !storePayloads;
}
@Override
@@ -64,6 +65,7 @@ final class FieldInfo {
void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector,
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) {
+
if (this.isIndexed != isIndexed) {
this.isIndexed = true; // once indexed, always index
}
@@ -85,7 +87,9 @@ final class FieldInfo {
}
if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) {
this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life
+ this.storePayloads = false;
}
}
+ assert !this.omitTermFreqAndPositions || !this.storePayloads;
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1092118&r1=1092117&r2=1092118&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FieldInfos.java Thu Apr 14 10:55:27 2011
@@ -240,6 +240,7 @@ final class FieldInfos {
} else {
fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
}
+ assert !fi.omitTermFreqAndPositions || !fi.storePayloads;
return fi;
}
@@ -321,6 +322,7 @@ final class FieldInfos {
output.writeVInt(size());
for (int i = 0; i < size(); i++) {
FieldInfo fi = fieldInfo(i);
+ assert !fi.omitTermFreqAndPositions || !fi.storePayloads;
byte bits = 0x0;
if (fi.isIndexed) bits |= IS_INDEXED;
if (fi.storeTermVector) bits |= STORE_TERMVECTOR;
@@ -367,6 +369,13 @@ final class FieldInfos {
boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
boolean omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0;
+ // LUCENE-3027: past indices were able to write
+ // storePayloads=true when omitTFAP is also true,
+ // which is invalid. We correct that, here:
+ if (omitTermFreqAndPositions) {
+ storePayloads = false;
+ }
+
addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1092118&r1=1092117&r2=1092118&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Thu Apr 14 10:55:27 2011
@@ -109,7 +109,9 @@ final class FreqProxTermsWriter extends
// Aggregate the storePayload as seen by the same
// field across multiple threads
- fieldInfo.storePayloads |= fields[i-start].hasPayloads;
+ if (!fieldInfo.omitTermFreqAndPositions) {
+ fieldInfo.storePayloads |= fields[i-start].hasPayloads;
+ }
}
// If this field has postings then add them to the
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1092118&r1=1092117&r2=1092118&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Thu Apr 14 10:55:27 2011
@@ -118,6 +118,7 @@ public class TestOmitTf extends LuceneTe
setMaxBufferedDocs(3).
setMergePolicy(newLogMergePolicy(2))
);
+ writer.setInfoStream(VERBOSE ? System.out : null);
Document d = new Document();
// this field will have Tf