You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/10/15 19:32:36 UTC
svn commit: r1632133 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/backward-codecs/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/or...
Author: mikemccand
Date: Wed Oct 15 17:32:35 2014
New Revision: 1632133
URL: http://svn.apache.org/r1632133
Log:
LUCENE-6006: remove unnecessary FieldInfo.normType
Added:
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip
- copied unchanged from r1632120, lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/backward-codecs/ (props changed)
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/branch_5x/lucene/codecs/ (props changed)
lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/branch_5x/lucene/memory/ (props changed)
lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/branch_5x/lucene/misc/ (props changed)
lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Wed Oct 15 17:32:35 2014
@@ -143,6 +143,10 @@ API Changes
their jflex impl directly.
(Ryan Ernst)
+* LUCENE-6006: Removed FieldInfo.normType since it's redundant: it
+ will be DocValuesType.NUMERIC if the field indexed and does not omit
+ norms, else null. (Robert Muir, Mike McCandless)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -91,7 +92,7 @@ public class Lucene40FieldInfosFormat ex
byte val = input.readByte();
final LegacyDocValuesType oldValuesType = getDocValuesType((byte) (val & 0x0F));
final LegacyDocValuesType oldNormsType = getDocValuesType((byte) ((val >>> 4) & 0x0F));
- final Map<String,String> attributes = input.readStringStringMap();;
+ final Map<String,String> attributes = input.readStringStringMap();
if (oldValuesType.mapping != null) {
attributes.put(LEGACY_DV_TYPE_KEY, oldValuesType.name());
}
@@ -101,8 +102,12 @@ public class Lucene40FieldInfosFormat ex
}
attributes.put(LEGACY_NORM_TYPE_KEY, oldNormsType.name());
}
+ if (isIndexed && omitNorms == false && oldNormsType.mapping == null) {
+ // Undead norms! Lucene40NormsReader will check this and bring norms back from the dead:
+ UndeadNormsProducer.setUndead(attributes);
+ }
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, oldValuesType.mapping, oldNormsType.mapping, -1, Collections.unmodifiableMap(attributes));
+ omitNorms, storePayloads, indexOptions, oldValuesType.mapping, -1, Collections.unmodifiableMap(attributes));
}
CodecUtil.checkEOF(input);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -46,6 +47,10 @@ public class Lucene40NormsFormat extends
String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
"nrm",
Lucene40CompoundFormat.COMPOUND_FILE_EXTENSION);
- return new Lucene40NormsReader(state, filename);
+ if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+ return UndeadNormsProducer.INSTANCE;
+ } else {
+ return new Lucene40NormsReader(state, filename);
+ }
}
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java Wed Oct 15 17:32:35 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +47,10 @@ final class Lucene40NormsReader extends
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (UndeadNormsProducer.isUndead(field)) {
+ // Bring undead norms back to life; this is set in Lucene40FieldInfosFormat, to emulate pre-5.0 undead norms
+ return DocValues.emptyNumeric();
+ }
return impl.getNumeric(field);
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -41,7 +42,7 @@ import org.apache.lucene.util.IOUtils;
*/
@Deprecated
public class Lucene42FieldInfosFormat extends FieldInfosFormat {
-
+
/** Sole constructor. */
public Lucene42FieldInfosFormat() {
}
@@ -86,8 +87,14 @@ public class Lucene42FieldInfosFormat ex
final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final Map<String,String> attributes = input.readStringStringMap();
+
+ if (isIndexed && omitNorms == false && normsType == null) {
+ // Undead norms! Lucene42NormsProducer will check this and bring norms back from the dead:
+ UndeadNormsProducer.setUndead(attributes);
+ }
+
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType, normsType, -1, Collections.unmodifiableMap(attributes));
+ omitNorms, storePayloads, indexOptions, docValuesType, -1, Collections.unmodifiableMap(attributes));
}
CodecUtil.checkEOF(input);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.packed.PackedInts;
@@ -62,7 +63,11 @@ public class Lucene42NormsFormat extends
@Override
public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
- return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+ return UndeadNormsProducer.INSTANCE;
+ } else {
+ return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
}
static final String DATA_CODEC = "Lucene41NormsData";
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java Wed Oct 15 17:32:35 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +47,10 @@ final class Lucene42NormsProducer extend
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (UndeadNormsProducer.isUndead(field)) {
+ // Bring undead norms back to life; this is set in Lucene42FieldInfosFormat, to emulate pre-5.0 undead norms
+ return DocValues.emptyNumeric();
+ }
return impl.getNumeric(field);
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -42,7 +43,7 @@ import org.apache.lucene.store.IndexOutp
*/
@Deprecated
public final class Lucene46FieldInfosFormat extends FieldInfosFormat {
-
+
/** Sole constructor. */
public Lucene46FieldInfosFormat() {
}
@@ -88,8 +89,14 @@ public final class Lucene46FieldInfosFor
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
+
+ if (isIndexed && omitNorms == false && normsType == null) {
+ // Undead norms! Lucene42NormsProducer will check this and bring norms back from the dead:
+ UndeadNormsProducer.setUndead(attributes);
+ }
+
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
+ omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
}
if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
@@ -148,7 +155,7 @@ public final class Lucene46FieldInfosFor
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
- final byte nrm = docValuesByte(fi.getNormType());
+ final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -42,7 +43,11 @@ public class Lucene49NormsFormat extends
@Override
public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
- return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+ return UndeadNormsProducer.INSTANCE;
+ } else {
+ return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
}
static final String DATA_CODEC = "Lucene49NormsData";
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java Wed Oct 15 17:32:35 2014
@@ -25,7 +25,9 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
@@ -40,8 +42,8 @@ import org.apache.lucene.util.RamUsageEs
import org.apache.lucene.util.packed.BlockPackedReader;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
/**
* Reader for 4.9 norms
@@ -153,6 +155,10 @@ final class Lucene49NormsProducer extend
@Override
public synchronized NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (UndeadNormsProducer.isUndead(field)) {
+ // Bring undead norms back to life; this is set in Lucene46FieldInfosFormat, to emulate pre-5.0 undead norms
+ return DocValues.emptyNumeric();
+ }
NumericDocValues instance = instances.get(field.name);
if (instance == null) {
instance = loadNorms(field);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -75,7 +75,7 @@ public final class Lucene40RWFieldInfosF
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType(), fi.getAttribute(LEGACY_DV_TYPE_KEY));
- final byte nrm = docValuesByte(fi.getNormType(), fi.getAttribute(LEGACY_NORM_TYPE_KEY));
+ final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null, fi.getAttribute(LEGACY_NORM_TYPE_KEY));
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java Wed Oct 15 17:32:35 2014
@@ -64,7 +64,7 @@ public class TestLucene40FieldInfoFormat
}
}
- if (fi.getNormType() != null) {
+ if (fi.hasNorms()) {
fi.putAttribute(Lucene40FieldInfosFormat.LEGACY_NORM_TYPE_KEY, LegacyDocValuesType.FIXED_INTS_8.name());
}
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.InputStream;
+import java.nio.file.Path;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
/** Tests Lucene40's norms format */
public class TestLucene40NormsFormat extends BaseNormsFormatTestCase {
@@ -28,4 +38,101 @@ public class TestLucene40NormsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Copy this back to /l/400/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.40.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(new File("/tmp/undeadnorms"));
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene40NormsFormat.class.getResourceAsStream("index.40.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -76,7 +76,7 @@ public final class Lucene42RWFieldInfosF
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
- final byte nrm = docValuesByte(fi.getNormType());
+ final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -17,10 +17,18 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.InputStream;
+import java.nio.file.Path;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
-import org.junit.BeforeClass;
-
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
/** Tests Lucene42's norms format */
public class TestLucene42NormsFormat extends BaseNormsFormatTestCase {
@@ -30,4 +38,100 @@ public class TestLucene42NormsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Copy this back to /l/421/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.42.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(file);
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_42, new WhitespaceAnalyzer(Version.LUCENE_42)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene42NormsFormat.class.getResourceAsStream("index.42.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java Wed Oct 15 17:32:35 2014
@@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.InputStream;
+import java.nio.file.Path;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
/**
* Tests Lucene49NormsFormat
@@ -30,4 +40,101 @@ public class TestLucene49NormsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Copy this back to /l/491/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.49.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(file);
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_4_9, new WhitespaceAnalyzer(Version.LUCENE_4_9)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene49NormsFormat.class.getResourceAsStream("index.49.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Wed Oct 15 17:32:35 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.lang.reflect.Modifier;
-import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -121,7 +120,7 @@ public class TestBackwardsCompatibility
private Path getIndexDir() {
String path = System.getProperty("tests.bwcdir");
- assumeTrue("backcompat creation tests must be run with -Dtests,bwcdir=/path/to/write/indexes", path != null);
+ assumeTrue("backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes", path != null);
return Paths.get(path);
}
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java Wed Oct 15 17:32:35 2014
@@ -71,8 +71,7 @@ class SimpleTextDocValuesWriter extends
@Override
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
assert fieldSeen(field.name);
- assert (field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC ||
- field.getNormType() == FieldInfo.DocValuesType.NUMERIC);
+ assert field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC || field.hasNorms();
writeFieldEntry(field, FieldInfo.DocValuesType.NUMERIC);
// first pass to find min/max
Modified: lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -59,7 +59,6 @@ public class SimpleTextFieldInfosFormat
static final BytesRef STORETVOFF = new BytesRef(" term vector offsets ");
static final BytesRef PAYLOADS = new BytesRef(" payloads ");
static final BytesRef NORMS = new BytesRef(" norms ");
- static final BytesRef NORMS_TYPE = new BytesRef(" norms type ");
static final BytesRef DOCVALUES = new BytesRef(" doc values ");
static final BytesRef DOCVALUES_GEN = new BytesRef(" doc values gen ");
static final BytesRef INDEXOPTIONS = new BytesRef(" index options ");
@@ -116,11 +115,6 @@ public class SimpleTextFieldInfosFormat
boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
SimpleTextUtil.readLine(input, scratch);
- assert StringHelper.startsWith(scratch.get(), NORMS_TYPE);
- String nrmType = readString(NORMS_TYPE.length, scratch);
- final DocValuesType normsType = docValuesType(nrmType);
-
- SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DOCVALUES);
String dvType = readString(DOCVALUES.length, scratch);
final DocValuesType docValuesType = docValuesType(dvType);
@@ -146,7 +140,7 @@ public class SimpleTextFieldInfosFormat
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(atts));
+ omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts));
}
SimpleTextUtil.checkFooter(input);
@@ -217,11 +211,7 @@ public class SimpleTextFieldInfosFormat
SimpleTextUtil.write(out, NORMS);
SimpleTextUtil.write(out, Boolean.toString(!fi.omitsNorms()), scratch);
SimpleTextUtil.writeNewline(out);
-
- SimpleTextUtil.write(out, NORMS_TYPE);
- SimpleTextUtil.write(out, getDocValuesType(fi.getNormType()), scratch);
- SimpleTextUtil.writeNewline(out);
-
+
SimpleTextUtil.write(out, DOCVALUES);
SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
SimpleTextUtil.writeNewline(out);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Wed Oct 15 17:32:35 2014
@@ -102,7 +102,7 @@ import org.apache.lucene.store.IndexOutp
* @lucene.experimental
*/
public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
-
+
/** Sole constructor. */
public Lucene50FieldInfosFormat() {
}
@@ -149,12 +149,11 @@ public final class Lucene50FieldInfosFor
// DV Types are packed in one byte
byte val = input.readByte();
final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
- final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
try {
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads,
- indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
+ indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@@ -215,12 +214,10 @@ public final class Lucene50FieldInfosFor
output.writeVInt(fi.number);
output.writeByte(bits);
- // pack the DV types in one byte
+ // pack the DV type and hasNorms in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
- final byte nrm = docValuesByte(fi.getNormType());
- assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
- byte val = (byte) (0xff & ((nrm << 4) | dv));
- output.writeByte(val);
+ assert (dv & (~0xF)) == 0;
+ output.writeByte(dv);
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Wed Oct 15 17:32:35 2014
@@ -1791,12 +1791,8 @@ public class CheckIndex implements Close
}
private static void checkNorms(FieldInfo fi, LeafReader reader, PrintStream infoStream) throws IOException {
- switch(fi.getNormType()) {
- case NUMERIC:
- checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
- break;
- default:
- throw new AssertionError("wtf: " + fi.getNormType());
+ if (fi.hasNorms()) {
+ checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
}
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Wed Oct 15 17:32:35 2014
@@ -183,14 +183,10 @@ final class DefaultIndexingChain extends
// we must check the final value of omitNorms for the fieldinfo: it could have
// changed for this field since the first time we added it.
- if (fi.omitsNorms() == false) {
- if (perField.norms != null) {
- perField.norms.finish(state.segmentInfo.getDocCount());
- perField.norms.flush(state, normsConsumer);
- assert fi.getNormType() == DocValuesType.NUMERIC;
- } else if (fi.isIndexed()) {
- assert fi.getNormType() == null: "got " + fi.getNormType() + "; field=" + fi.name;
- }
+ if (fi.omitsNorms() == false && fi.isIndexed()) {
+ assert perField.norms != null: "field=" + fi.name;
+ perField.norms.finish(state.segmentInfo.getDocCount());
+ perField.norms.flush(state, normsConsumer);
}
}
}
@@ -546,6 +542,11 @@ final class DefaultIndexingChain extends
void setInvertState() {
invertState = new FieldInvertState(fieldInfo.name);
termsHashPerField = termsHash.addField(invertState, fieldInfo);
+ if (fieldInfo.omitsNorms() == false) {
+ assert norms == null;
+ // Even if no documents actually succeed in setting a norm, we still write norms for this segment:
+ norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
+ }
}
@Override
@@ -554,14 +555,8 @@ final class DefaultIndexingChain extends
}
public void finish() throws IOException {
- if (fieldInfo.omitsNorms() == false) {
- if (norms == null) {
- fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
- norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
- }
- if (invertState.length != 0) {
- norms.addValue(docState.docID, similarity.computeNorm(invertState));
- }
+ if (fieldInfo.omitsNorms() == false && invertState.length != 0) {
+ norms.addValue(docState.docID, similarity.computeNorm(invertState));
}
termsHashPerField.finish();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Wed Oct 15 17:32:35 2014
@@ -40,15 +40,15 @@ public final class FieldInfo {
// True if any document indexed term vectors
private boolean storeTermVector;
- private DocValuesType normType;
private boolean omitNorms; // omit norms associated with indexed fields
+
private IndexOptions indexOptions;
private boolean storePayloads; // whether this field stores payloads together with term positions
private Map<String,String> attributes;
private long dvGen;
-
+
/**
* Controls how much information is stored in the postings lists.
* @lucene.experimental
@@ -120,12 +120,12 @@ public final class FieldInfo {
}
/**
- * Sole Constructor.
+ * Sole constructor.
*
* @lucene.experimental
*/
public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector, boolean omitNorms,
- boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normsType,
+ boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
long dvGen, Map<String,String> attributes) {
this.name = name;
this.indexed = indexed;
@@ -136,13 +136,11 @@ public final class FieldInfo {
this.storePayloads = storePayloads;
this.omitNorms = omitNorms;
this.indexOptions = indexOptions;
- this.normType = !omitNorms ? normsType : null;
} else { // for non-indexed fields, leave defaults
this.storeTermVector = false;
this.storePayloads = false;
this.omitNorms = false;
this.indexOptions = null;
- this.normType = null;
}
this.dvGen = dvGen;
this.attributes = attributes;
@@ -158,11 +156,6 @@ public final class FieldInfo {
if (indexOptions == null) {
throw new IllegalStateException("indexed field '" + name + "' must have index options");
}
- if (omitNorms) {
- if (normType != null) {
- throw new IllegalStateException("indexed field '" + name + "' cannot both omit norms and have norms");
- }
- }
// Cannot store payloads unless positions are indexed:
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0 && storePayloads) {
throw new IllegalStateException("indexed field '" + name + "' cannot have payloads without positions");
@@ -177,12 +170,8 @@ public final class FieldInfo {
if (omitNorms) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot omit norms");
}
- if (normType != null) {
- throw new IllegalStateException("non-indexed field '" + name + "' cannot have norms");
- }
if (indexOptions != null) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot have index options");
-
}
}
@@ -206,7 +195,6 @@ public final class FieldInfo {
this.storePayloads |= storePayloads;
if (this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
- this.normType = null;
}
if (this.indexOptions != indexOptions) {
if (this.indexOptions == null) {
@@ -265,13 +253,6 @@ public final class FieldInfo {
return dvGen;
}
- /**
- * Returns {@link DocValuesType} of the norm. this may be null if the field has no norms.
- */
- public DocValuesType getNormType() {
- return normType;
- }
-
void setStoreTermVectors() {
storeTermVector = true;
assert checkConsistency();
@@ -284,14 +265,6 @@ public final class FieldInfo {
assert checkConsistency();
}
- void setNormValueType(DocValuesType type) {
- if (normType != null && normType != type) {
- throw new IllegalArgumentException("cannot change Norm type from " + normType + " to " + type + " for field \"" + name + "\"");
- }
- normType = type;
- assert checkConsistency();
- }
-
/**
* Returns true if norms are explicitly omitted for this field
*/
@@ -303,7 +276,7 @@ public final class FieldInfo {
* Returns true if this field actually has any norms.
*/
public boolean hasNorms() {
- return normType != null;
+ return indexed && omitNorms == false;
}
/**
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Wed Oct 15 17:32:35 2014
@@ -276,10 +276,8 @@ public class FieldInfos implements Itera
}
/** NOTE: this method does not carry over termVector
- * booleans nor docValuesType; the indexer chain
- * (TermVectorsConsumerPerField, DocFieldProcessor) must
- * set these fields when they succeed in consuming
- * the document */
+ * the indexer chain must set these fields when they
+ * succeed in consuming the document */
public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
// TODO: really, indexer shouldn't even call this
// method (it's only called from DocFieldProcessor);
@@ -288,12 +286,12 @@ public class FieldInfos implements Itera
// be updated by maybe FreqProxTermsWriterPerField:
return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
fieldType.omitNorms(), false,
- fieldType.indexOptions(), fieldType.docValueType(), null);
+ fieldType.indexOptions(), fieldType.docValueType());
}
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
boolean storeTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normType) {
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
FieldInfo fi = fieldInfo(name);
if (fi == null) {
// This field wasn't yet added to this in-RAM
@@ -302,7 +300,7 @@ public class FieldInfos implements Itera
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
- fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType, -1, null);
+ fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
assert !byName.containsKey(fi.name);
assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
@@ -319,26 +317,22 @@ public class FieldInfos implements Itera
globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
}
}
-
- if (!fi.omitsNorms() && normType != null) {
- fi.setNormValueType(normType);
- }
}
return fi;
}
-
+
public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
- fi.getIndexOptions(), fi.getDocValuesType(), fi.getNormType());
+ fi.getIndexOptions(), fi.getDocValuesType());
}
public FieldInfo fieldInfo(String fieldName) {
return byName.get(fieldName);
}
- final FieldInfos finish() {
+ FieldInfos finish() {
return new FieldInfos(byName.values().toArray(new FieldInfo[byName.size()]));
}
}
Modified: lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_5x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Wed Oct 15 17:32:35 2014
@@ -445,7 +445,7 @@ public class MemoryIndex {
if (!fieldInfos.containsKey(fieldName)) {
fieldInfos.put(fieldName,
- new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, null, -1, null));
+ new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, -1, null));
}
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
Modified: lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java Wed Oct 15 17:32:35 2014
@@ -21,22 +21,22 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
+import org.apache.lucene.document.BinaryDocValuesField; // javadocs
+import org.apache.lucene.document.DoubleField; // javadocs
+import org.apache.lucene.document.FloatField; // javadocs
import org.apache.lucene.document.IntField; // javadocs
import org.apache.lucene.document.LongField; // javadocs
-import org.apache.lucene.document.FloatField; // javadocs
-import org.apache.lucene.document.DoubleField; // javadocs
-import org.apache.lucene.document.BinaryDocValuesField; // javadocs
import org.apache.lucene.document.NumericDocValuesField; // javadocs
import org.apache.lucene.document.SortedDocValuesField; // javadocs
import org.apache.lucene.document.SortedSetDocValuesField; // javadocs
import org.apache.lucene.document.StringField; // javadocs
-import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@@ -213,7 +213,7 @@ public class UninvertingReader extends F
}
}
filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
- fi.hasPayloads(), fi.getIndexOptions(), type, fi.getNormType(), -1, null));
+ fi.hasPayloads(), fi.getIndexOptions(), type, -1, null));
}
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Wed Oct 15 17:32:35 2014
@@ -93,7 +93,7 @@ public class AssertingNormsFormat extend
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
- assert field.getNormType() == FieldInfo.DocValuesType.NUMERIC;
+ assert field.hasNorms();
NumericDocValues values = in.getNorms(field);
assert values != null;
return new AssertingLeafReader.AssertingNumericDocValues(values, maxDoc);
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Wed Oct 15 17:32:35 2014
@@ -89,11 +89,6 @@ public abstract class BaseFieldInfoForma
fi.setStorePayloads();
}
}
- if (fi.isIndexed() && !fi.omitsNorms()) {
- if (random().nextBoolean()) {
- fi.setNormValueType(DocValuesType.NUMERIC);
- }
- }
addAttributes(fi);
}
FieldInfos infos = builder.finish();
@@ -165,7 +160,6 @@ public abstract class BaseFieldInfoForma
assertEquals(expected.name, actual.name);
assertEquals(expected.getDocValuesType(), actual.getDocValuesType());
assertEquals(expected.getIndexOptions(), actual.getIndexOptions());
- assertEquals(expected.getNormType(), actual.getNormType());
assertEquals(expected.hasDocValues(), actual.hasDocValues());
assertEquals(expected.hasNorms(), actual.hasNorms());
assertEquals(expected.hasPayloads(), actual.hasPayloads());
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Wed Oct 15 17:32:35 2014
@@ -18,6 +18,8 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -128,7 +130,6 @@ public abstract class BaseNormsFormatTes
public void testAllZeros() throws Exception {
int iterations = atLeast(1);
- final Random r = random();
for (int i = 0; i < iterations; i++) {
doTestNormsVersusStoredFields(new LongProducer() {
@Override
@@ -264,4 +265,60 @@ public abstract class BaseNormsFormatTes
}
// TODO: test thread safety (e.g. across different fields) explicitly here
+
+ /**
+ * LUCENE-6006: Tests undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testUndeadNorms() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ int numDocs = atLeast(1000);
+ List<Integer> toDelete = new ArrayList<>();
+ for(int i=0;i<numDocs;i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", ""+i, Field.Store.NO));
+ if (random().nextInt(5) == 1) {
+ toDelete.add(i);
+ doc.add(new TextField("content", "some content", Field.Store.NO));
+ }
+ w.addDocument(doc);
+ }
+ for(Integer id : toDelete) {
+ w.deleteDocuments(new Term("id", ""+id));
+ }
+ w.forceMerge(1);
+ IndexReader r = w.getReader();
+
+ // Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not
+ // be undead:
+ NumericDocValues norms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(norms);
+ for(int i=0;i<r.maxDoc();i++) {
+ assertEquals(0, norms.get(i));
+ }
+
+ r.close();
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Wed Oct 15 17:32:35 2014
@@ -45,7 +45,6 @@ import org.apache.lucene.codecs.perfield
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.Directory;
@@ -373,7 +372,7 @@ public abstract class BasePostingsFormat
fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
- null, DocValuesType.NUMERIC, -1, null);
+ null, -1, null);
fieldUpto++;
SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@@ -702,7 +701,6 @@ public abstract class BasePostingsFormat
doPayloads,
indexOptions,
null,
- DocValuesType.NUMERIC,
-1,
null);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java?rev=1632133&r1=1632132&r2=1632133&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/search/Insanity.java Wed Oct 15 17:32:35 2014
@@ -65,7 +65,7 @@ public class Insanity {
for (FieldInfo fi : in.getFieldInfos()) {
if (fi.name.equals(insaneField)) {
filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
- fi.hasPayloads(), fi.getIndexOptions(), null, fi.getNormType(), -1, null));
+ fi.hasPayloads(), fi.getIndexOptions(), null, -1, null));
} else {
filteredInfos.add(fi);
}