You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/10/15 18:58:31 UTC
svn commit: r1632120 - in /lucene/dev/trunk: lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/ lucene/...
Author: mikemccand
Date: Wed Oct 15 16:58:30 2014
New Revision: 1632120
URL: http://svn.apache.org/r1632120
Log:
LUCENE-6006: remove unnecessary FieldInfo.normType
Added:
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java (with props)
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html (with props)
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip (with props)
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip (with props)
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java (with props)
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip (with props)
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip (with props)
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/Insanity.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Wed Oct 15 16:58:30 2014
@@ -160,6 +160,10 @@ API Changes
their jflex impl directly.
(Ryan Ernst)
+* LUCENE-6006: Removed FieldInfo.normType since it's redundant: it
+ will be DocValuesType.NUMERIC if the field indexed and does not omit
+ norms, else null. (Robert Muir, Mike McCandless)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
Added: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java?rev=1632120&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java (added)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/UndeadNormsProducer.java Wed Oct 15 16:58:30 2014
@@ -0,0 +1,113 @@
+package org.apache.lucene.codecs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.util.Accountable;
+
+/**
+ * Used only for backwards compatibility corner case, to provide
+ * re-animated norms when all fields are undead.
+ *
+ * @lucene.internal */
+public class UndeadNormsProducer extends NormsProducer {
+
+ /** Used to bring undead norms back to life. */
+ public final static String LEGACY_UNDEAD_NORMS_KEY = UndeadNormsProducer.class.getSimpleName() + ".undeadnorms";
+
+ /** Use this instance */
+ public final static NormsProducer INSTANCE = new UndeadNormsProducer();
+
+ private UndeadNormsProducer() {
+ }
+
+ /* Returns true if all indexed fields have undead norms. */
+ public static boolean isUndeadArmy(FieldInfos fieldInfos) {
+
+ boolean everythingIsUndead = true;
+ for(FieldInfo fieldInfo : fieldInfos) {
+ if (fieldInfo.hasNorms()) {
+ String isUndead = fieldInfo.getAttribute(LEGACY_UNDEAD_NORMS_KEY);
+ if (isUndead != null) {
+ assert "true".equals(isUndead);
+ } else {
+ everythingIsUndead = false;
+ }
+ }
+ }
+
+ return everythingIsUndead;
+ }
+
+ /** Returns true if this field has undead norms. */
+ public static boolean isUndead(FieldInfo fieldInfo) {
+ String isUndead = fieldInfo.getAttribute(LEGACY_UNDEAD_NORMS_KEY);
+ if (isUndead != null) {
+ // Bring undead norms back to life; this is set in Lucene40FieldInfosFormat, to emulate pre-5.0 undead norms
+ assert "true".equals(isUndead);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ /** Call this to note that the field with these attributes has undead norms. */
+ public static void setUndead(Map<String,String> attributes) {
+ attributes.put(LEGACY_UNDEAD_NORMS_KEY, "true");
+ }
+
+ @Override
+ public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ return DocValues.emptyNumeric();
+ }
+
+ @Override
+ public void close() {
+ }
+
+ @Override
+ public long ramBytesUsed() {
+ return 0;
+ }
+
+ @Override
+ public Iterable<? extends Accountable> getChildResources() {
+ return Collections.emptyList();
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ }
+
+ @Override
+ public NormsProducer getMergeInstance() throws IOException {
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName();
+ }
+}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40FieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -91,7 +92,7 @@ public class Lucene40FieldInfosFormat ex
byte val = input.readByte();
final LegacyDocValuesType oldValuesType = getDocValuesType((byte) (val & 0x0F));
final LegacyDocValuesType oldNormsType = getDocValuesType((byte) ((val >>> 4) & 0x0F));
- final Map<String,String> attributes = input.readStringStringMap();;
+ final Map<String,String> attributes = input.readStringStringMap();
if (oldValuesType.mapping != null) {
attributes.put(LEGACY_DV_TYPE_KEY, oldValuesType.name());
}
@@ -101,8 +102,12 @@ public class Lucene40FieldInfosFormat ex
}
attributes.put(LEGACY_NORM_TYPE_KEY, oldNormsType.name());
}
+ if (isIndexed && omitNorms == false && oldNormsType.mapping == null) {
+ // Undead norms! Lucene40NormsReader will check this and bring norms back from the dead:
+ UndeadNormsProducer.setUndead(attributes);
+ }
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, oldValuesType.mapping, oldNormsType.mapping, -1, Collections.unmodifiableMap(attributes));
+ omitNorms, storePayloads, indexOptions, oldValuesType.mapping, -1, Collections.unmodifiableMap(attributes));
}
CodecUtil.checkEOF(input);
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsFormat.java Wed Oct 15 16:58:30 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -46,6 +47,10 @@ public class Lucene40NormsFormat extends
String filename = IndexFileNames.segmentFileName(state.segmentInfo.name,
"nrm",
Lucene40CompoundFormat.COMPOUND_FILE_EXTENSION);
- return new Lucene40NormsReader(state, filename);
+ if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+ return UndeadNormsProducer.INSTANCE;
+ } else {
+ return new Lucene40NormsReader(state, filename);
+ }
}
}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40NormsReader.java Wed Oct 15 16:58:30 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +47,10 @@ final class Lucene40NormsReader extends
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (UndeadNormsProducer.isUndead(field)) {
+ // Bring undead norms back to life; this is set in Lucene40FieldInfosFormat, to emulate pre-5.0 undead norms
+ return DocValues.emptyNumeric();
+ }
return impl.getNumeric(field);
}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -41,7 +42,7 @@ import org.apache.lucene.util.IOUtils;
*/
@Deprecated
public class Lucene42FieldInfosFormat extends FieldInfosFormat {
-
+
/** Sole constructor. */
public Lucene42FieldInfosFormat() {
}
@@ -86,8 +87,14 @@ public class Lucene42FieldInfosFormat ex
final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final Map<String,String> attributes = input.readStringStringMap();
+
+ if (isIndexed && omitNorms == false && normsType == null) {
+ // Undead norms! Lucene42NormsProducer will check this and bring norms back from the dead:
+ UndeadNormsProducer.setUndead(attributes);
+ }
+
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType, normsType, -1, Collections.unmodifiableMap(attributes));
+ omitNorms, storePayloads, indexOptions, docValuesType, -1, Collections.unmodifiableMap(attributes));
}
CodecUtil.checkEOF(input);
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java Wed Oct 15 16:58:30 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.packed.PackedInts;
@@ -62,7 +63,11 @@ public class Lucene42NormsFormat extends
@Override
public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
- return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+ return UndeadNormsProducer.INSTANCE;
+ } else {
+ return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
}
static final String DATA_CODEC = "Lucene41NormsData";
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java Wed Oct 15 16:58:30 2014
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentReadState;
@@ -45,6 +47,10 @@ final class Lucene42NormsProducer extend
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (UndeadNormsProducer.isUndead(field)) {
+ // Bring undead norms back to life; this is set in Lucene42FieldInfosFormat, to emulate pre-5.0 undead norms
+ return DocValues.emptyNumeric();
+ }
return impl.getNumeric(field);
}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene46/Lucene46FieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -23,13 +23,14 @@ import java.util.Map;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -42,7 +43,7 @@ import org.apache.lucene.store.IndexOutp
*/
@Deprecated
public final class Lucene46FieldInfosFormat extends FieldInfosFormat {
-
+
/** Sole constructor. */
public Lucene46FieldInfosFormat() {
}
@@ -88,8 +89,14 @@ public final class Lucene46FieldInfosFor
final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
+
+ if (isIndexed && omitNorms == false && normsType == null) {
+ // Undead norms! Lucene42NormsProducer will check this and bring norms back from the dead:
+ UndeadNormsProducer.setUndead(attributes);
+ }
+
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
+ omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
}
if (codecVersion >= Lucene46FieldInfosFormat.FORMAT_CHECKSUM) {
@@ -148,7 +155,7 @@ public final class Lucene46FieldInfosFor
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
- final byte nrm = docValuesByte(fi.getNormType());
+ final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsFormat.java Wed Oct 15 16:58:30 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -42,7 +43,11 @@ public class Lucene49NormsFormat extends
@Override
public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
- return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ if (UndeadNormsProducer.isUndeadArmy(state.fieldInfos)) {
+ return UndeadNormsProducer.INSTANCE;
+ } else {
+ return new Lucene49NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
+ }
}
static final String DATA_CODEC = "Lucene49NormsData";
Modified: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene49/Lucene49NormsProducer.java Wed Oct 15 16:58:30 2014
@@ -25,7 +25,9 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.NormsProducer;
+import org.apache.lucene.codecs.UndeadNormsProducer;
import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
@@ -40,8 +42,8 @@ import org.apache.lucene.util.RamUsageEs
import org.apache.lucene.util.packed.BlockPackedReader;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene49.Lucene49NormsFormat.VERSION_START;
/**
* Reader for 4.9 norms
@@ -153,6 +155,10 @@ final class Lucene49NormsProducer extend
@Override
public synchronized NumericDocValues getNorms(FieldInfo field) throws IOException {
+ if (UndeadNormsProducer.isUndead(field)) {
+ // Bring undead norms back to life; this is set in Lucene46FieldInfosFormat, to emulate pre-5.0 undead norms
+ return DocValues.emptyNumeric();
+ }
NumericDocValues instance = instances.get(field.name);
if (instance == null) {
instance = loadNorms(field);
Added: lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html?rev=1632120&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html (added)
+++ lucene/dev/trunk/lucene/backward-codecs/src/java/org/apache/lucene/codecs/package.html Wed Oct 15 16:58:30 2014
@@ -0,0 +1,25 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+Common APIs for use by backwards compatibility codecs.
+</body>
+</html>
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/Lucene40RWFieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -75,7 +75,7 @@ public final class Lucene40RWFieldInfosF
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType(), fi.getAttribute(LEGACY_DV_TYPE_KEY));
- final byte nrm = docValuesByte(fi.getNormType(), fi.getAttribute(LEGACY_NORM_TYPE_KEY));
+ final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null, fi.getAttribute(LEGACY_NORM_TYPE_KEY));
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40FieldInfoFormat.java Wed Oct 15 16:58:30 2014
@@ -64,7 +64,7 @@ public class TestLucene40FieldInfoFormat
}
}
- if (fi.getNormType() != null) {
+ if (fi.hasNorms()) {
fi.putAttribute(Lucene40FieldInfosFormat.LEGACY_NORM_TYPE_KEY, LegacyDocValuesType.FIXED_INTS_8.name());
}
}
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/TestLucene40NormsFormat.java Wed Oct 15 16:58:30 2014
@@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.InputStream;
+import java.nio.file.Path;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
/** Tests Lucene40's norms format */
public class TestLucene40NormsFormat extends BaseNormsFormatTestCase {
@@ -28,4 +38,101 @@ public class TestLucene40NormsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Copy this back to /l/400/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.0-SNAPSHOT.jar:build/core/lucene-core-4.0-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.40.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(new File("/tmp/undeadnorms"));
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_40, new WhitespaceAnalyzer(Version.LUCENE_40)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene40NormsFormat.class.getResourceAsStream("index.40.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
}
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene40/index.40.undeadnorms.zip?rev=1632120&view=auto
==============================================================================
Binary file - no diff available.
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/Lucene42RWFieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -76,7 +76,7 @@ public final class Lucene42RWFieldInfosF
// pack the DV types in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
- final byte nrm = docValuesByte(fi.getNormType());
+ final byte nrm = docValuesByte(fi.hasNorms() ? DocValuesType.NUMERIC : null);
assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
byte val = (byte) (0xff & ((nrm << 4) | dv));
output.writeByte(val);
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/TestLucene42NormsFormat.java Wed Oct 15 16:58:30 2014
@@ -17,10 +17,18 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.InputStream;
+import java.nio.file.Path;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
-import org.junit.BeforeClass;
-
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
/** Tests Lucene42's norms format */
public class TestLucene42NormsFormat extends BaseNormsFormatTestCase {
@@ -30,4 +38,100 @@ public class TestLucene42NormsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Copy this back to /l/421/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.2.1-SNAPSHOT.jar:build/core/lucene-core-4.2.1-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.42.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(file);
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_42, new WhitespaceAnalyzer(Version.LUCENE_42)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene42NormsFormat.class.getResourceAsStream("index.42.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
}
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene42/index.42.undeadnorms.zip?rev=1632120&view=auto
==============================================================================
Binary file - no diff available.
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java?rev=1632120&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java (added)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/TestLucene46UndeadNorms.java Wed Oct 15 16:58:30 2014
@@ -0,0 +1,129 @@
+package org.apache.lucene.codecs.lucene46;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.InputStream;
+import java.nio.file.Path;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+public class TestLucene46UndeadNorms extends LuceneTestCase {
+
+ /** Copy this back to /l/461/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.6-SNAPSHOT.jar:build/core/lucene-core-4.6-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.6-SNAPSHOT.jar:build/core/lucene-core-4.6-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.46.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(file);
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_46, new WhitespaceAnalyzer(Version.LUCENE_46)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene46UndeadNorms.class.getResourceAsStream("index.46.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
+}
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene46/index.46.undeadnorms.zip?rev=1632120&view=auto
==============================================================================
Binary file - no diff available.
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/TestLucene49NormsFormat.java Wed Oct 15 16:58:30 2014
@@ -17,8 +17,18 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
+import java.io.InputStream;
+import java.nio.file.Path;
+
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.index.BaseNormsFormatTestCase;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiDocValues;
+import org.apache.lucene.index.NumericDocValues;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.TestUtil;
/**
* Tests Lucene49NormsFormat
@@ -30,4 +40,101 @@ public class TestLucene49NormsFormat ext
protected Codec getCodec() {
return codec;
}
+
+ /** Copy this back to /l/491/lucene/CreateUndeadNorms.java, then:
+ * - ant clean
+ * - pushd analysis/common; ant jar; popd
+ * - pushd core; ant jar; popd
+ * - javac -cp build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms.java
+ * - java -cp .:build/analysis/common/lucene-analyzers-common-4.9-SNAPSHOT.jar:build/core/lucene-core-4.9-SNAPSHOT.jar CreateUndeadNorms
+ * - cd /tmp/undeadnorms ; zip index.49.undeadnorms.zip *
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.Version;
+
+public class CreateUndeadNorms {
+ public static void main(String[] args) throws Exception {
+ File file = new File("/tmp/undeadnorms");
+ if (file.exists()) {
+ throw new RuntimeException("please remove /tmp/undeadnorms first");
+ }
+ Directory dir = FSDirectory.open(file);
+ IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_4_9, new WhitespaceAnalyzer(Version.LUCENE_4_9)));
+ Document doc = new Document();
+ doc.add(new StringField("id", "0", Field.Store.NO));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(new StringField("id", "1", Field.Store.NO));
+ Field content = new TextField("content", "some content", Field.Store.NO);
+ content.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() throws IOException {
+ throw new IOException("brains brains!");
+ }
+ });
+
+ doc.add(content);
+ try {
+ w.addDocument(doc);
+ throw new RuntimeException("didn't hit exception");
+ } catch (IOException ioe) {
+ // perfect
+ }
+ w.close();
+ dir.close();
+ }
+}
+*/
+
+ /**
+ * LUCENE-6006: Test undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testReadUndeadNorms() throws Exception {
+ InputStream resource = TestLucene49NormsFormat.class.getResourceAsStream("index.49.undeadnorms.zip");
+ assertNotNull(resource);
+ Path path = createTempDir("undeadnorms");
+ TestUtil.unzip(resource, path);
+ Directory dir = FSDirectory.open(path);
+ IndexReader r = DirectoryReader.open(dir);
+ NumericDocValues undeadNorms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(undeadNorms);
+ assertEquals(2, r.maxDoc());
+ assertEquals(0, undeadNorms.get(0));
+ assertEquals(0, undeadNorms.get(1));
+ dir.close();
+ r.close();
+ }
}
Added: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/codecs/lucene49/index.49.undeadnorms.zip?rev=1632120&view=auto
==============================================================================
Binary file - no diff available.
Modified: lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/trunk/lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Wed Oct 15 16:58:30 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
import java.lang.reflect.Modifier;
-import java.net.URL;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -121,7 +120,7 @@ public class TestBackwardsCompatibility
private Path getIndexDir() {
String path = System.getProperty("tests.bwcdir");
- assumeTrue("backcompat creation tests must be run with -Dtests,bwcdir=/path/to/write/indexes", path != null);
+ assumeTrue("backcompat creation tests must be run with -Dtests.bwcdir=/path/to/write/indexes", path != null);
return Paths.get(path);
}
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextDocValuesWriter.java Wed Oct 15 16:58:30 2014
@@ -71,8 +71,7 @@ class SimpleTextDocValuesWriter extends
@Override
public void addNumericField(FieldInfo field, Iterable<Number> values) throws IOException {
assert fieldSeen(field.name);
- assert (field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC ||
- field.getNormType() == FieldInfo.DocValuesType.NUMERIC);
+ assert field.getDocValuesType() == FieldInfo.DocValuesType.NUMERIC || field.hasNorms();
writeFieldEntry(field, FieldInfo.DocValuesType.NUMERIC);
// first pass to find min/max
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -59,7 +59,6 @@ public class SimpleTextFieldInfosFormat
static final BytesRef STORETVOFF = new BytesRef(" term vector offsets ");
static final BytesRef PAYLOADS = new BytesRef(" payloads ");
static final BytesRef NORMS = new BytesRef(" norms ");
- static final BytesRef NORMS_TYPE = new BytesRef(" norms type ");
static final BytesRef DOCVALUES = new BytesRef(" doc values ");
static final BytesRef DOCVALUES_GEN = new BytesRef(" doc values gen ");
static final BytesRef INDEXOPTIONS = new BytesRef(" index options ");
@@ -116,11 +115,6 @@ public class SimpleTextFieldInfosFormat
boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
SimpleTextUtil.readLine(input, scratch);
- assert StringHelper.startsWith(scratch.get(), NORMS_TYPE);
- String nrmType = readString(NORMS_TYPE.length, scratch);
- final DocValuesType normsType = docValuesType(nrmType);
-
- SimpleTextUtil.readLine(input, scratch);
assert StringHelper.startsWith(scratch.get(), DOCVALUES);
String dvType = readString(DOCVALUES.length, scratch);
final DocValuesType docValuesType = docValuesType(dvType);
@@ -146,7 +140,7 @@ public class SimpleTextFieldInfosFormat
}
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector,
- omitNorms, storePayloads, indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(atts));
+ omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts));
}
SimpleTextUtil.checkFooter(input);
@@ -217,11 +211,7 @@ public class SimpleTextFieldInfosFormat
SimpleTextUtil.write(out, NORMS);
SimpleTextUtil.write(out, Boolean.toString(!fi.omitsNorms()), scratch);
SimpleTextUtil.writeNewline(out);
-
- SimpleTextUtil.write(out, NORMS_TYPE);
- SimpleTextUtil.write(out, getDocValuesType(fi.getNormType()), scratch);
- SimpleTextUtil.writeNewline(out);
-
+
SimpleTextUtil.write(out, DOCVALUES);
SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
SimpleTextUtil.writeNewline(out);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene50/Lucene50FieldInfosFormat.java Wed Oct 15 16:58:30 2014
@@ -102,7 +102,7 @@ import org.apache.lucene.store.IndexOutp
* @lucene.experimental
*/
public final class Lucene50FieldInfosFormat extends FieldInfosFormat {
-
+
/** Sole constructor. */
public Lucene50FieldInfosFormat() {
}
@@ -149,12 +149,11 @@ public final class Lucene50FieldInfosFor
// DV Types are packed in one byte
byte val = input.readByte();
final DocValuesType docValuesType = getDocValuesType(input, (byte) (val & 0x0F));
- final DocValuesType normsType = getDocValuesType(input, (byte) ((val >>> 4) & 0x0F));
final long dvGen = input.readLong();
final Map<String,String> attributes = input.readStringStringMap();
try {
infos[i] = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads,
- indexOptions, docValuesType, normsType, dvGen, Collections.unmodifiableMap(attributes));
+ indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(attributes));
infos[i].checkConsistency();
} catch (IllegalStateException e) {
throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
@@ -215,12 +214,10 @@ public final class Lucene50FieldInfosFor
output.writeVInt(fi.number);
output.writeByte(bits);
- // pack the DV types in one byte
+ // pack the DV type and hasNorms in one byte
final byte dv = docValuesByte(fi.getDocValuesType());
- final byte nrm = docValuesByte(fi.getNormType());
- assert (dv & (~0xF)) == 0 && (nrm & (~0x0F)) == 0;
- byte val = (byte) (0xff & ((nrm << 4) | dv));
- output.writeByte(val);
+ assert (dv & (~0xF)) == 0;
+ output.writeByte(dv);
output.writeLong(fi.getDocValuesGen());
output.writeStringStringMap(fi.attributes());
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Wed Oct 15 16:58:30 2014
@@ -1790,12 +1790,8 @@ public class CheckIndex implements Close
}
private static void checkNorms(FieldInfo fi, LeafReader reader, PrintStream infoStream) throws IOException {
- switch(fi.getNormType()) {
- case NUMERIC:
- checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
- break;
- default:
- throw new AssertionError("wtf: " + fi.getNormType());
+ if (fi.hasNorms()) {
+ checkNumericDocValues(fi.name, reader, reader.getNormValues(fi.name), new Bits.MatchAllBits(reader.maxDoc()));
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Wed Oct 15 16:58:30 2014
@@ -183,14 +183,10 @@ final class DefaultIndexingChain extends
// we must check the final value of omitNorms for the fieldinfo: it could have
// changed for this field since the first time we added it.
- if (fi.omitsNorms() == false) {
- if (perField.norms != null) {
- perField.norms.finish(state.segmentInfo.getDocCount());
- perField.norms.flush(state, normsConsumer);
- assert fi.getNormType() == DocValuesType.NUMERIC;
- } else if (fi.isIndexed()) {
- assert fi.getNormType() == null: "got " + fi.getNormType() + "; field=" + fi.name;
- }
+ if (fi.omitsNorms() == false && fi.isIndexed()) {
+ assert perField.norms != null: "field=" + fi.name;
+ perField.norms.finish(state.segmentInfo.getDocCount());
+ perField.norms.flush(state, normsConsumer);
}
}
}
@@ -535,6 +531,11 @@ final class DefaultIndexingChain extends
void setInvertState() {
invertState = new FieldInvertState(fieldInfo.name);
termsHashPerField = termsHash.addField(invertState, fieldInfo);
+ if (fieldInfo.omitsNorms() == false) {
+ assert norms == null;
+ // Even if no documents actually succeed in setting a norm, we still write norms for this segment:
+ norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
+ }
}
@Override
@@ -543,14 +544,8 @@ final class DefaultIndexingChain extends
}
public void finish() throws IOException {
- if (fieldInfo.omitsNorms() == false) {
- if (norms == null) {
- fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
- norms = new NormValuesWriter(fieldInfo, docState.docWriter.bytesUsed);
- }
- if (invertState.length != 0) {
- norms.addValue(docState.docID, similarity.computeNorm(invertState));
- }
+ if (fieldInfo.omitsNorms() == false && invertState.length != 0) {
+ norms.addValue(docState.docID, similarity.computeNorm(invertState));
}
termsHashPerField.finish();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Wed Oct 15 16:58:30 2014
@@ -40,15 +40,15 @@ public final class FieldInfo {
// True if any document indexed term vectors
private boolean storeTermVector;
- private DocValuesType normType;
private boolean omitNorms; // omit norms associated with indexed fields
+
private IndexOptions indexOptions;
private boolean storePayloads; // whether this field stores payloads together with term positions
private Map<String,String> attributes;
private long dvGen;
-
+
/**
* Controls how much information is stored in the postings lists.
* @lucene.experimental
@@ -120,12 +120,12 @@ public final class FieldInfo {
}
/**
- * Sole Constructor.
+ * Sole constructor.
*
* @lucene.experimental
*/
public FieldInfo(String name, boolean indexed, int number, boolean storeTermVector, boolean omitNorms,
- boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normsType,
+ boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
long dvGen, Map<String,String> attributes) {
this.name = name;
this.indexed = indexed;
@@ -136,13 +136,11 @@ public final class FieldInfo {
this.storePayloads = storePayloads;
this.omitNorms = omitNorms;
this.indexOptions = indexOptions;
- this.normType = !omitNorms ? normsType : null;
} else { // for non-indexed fields, leave defaults
this.storeTermVector = false;
this.storePayloads = false;
this.omitNorms = false;
this.indexOptions = null;
- this.normType = null;
}
this.dvGen = dvGen;
this.attributes = attributes;
@@ -158,11 +156,6 @@ public final class FieldInfo {
if (indexOptions == null) {
throw new IllegalStateException("indexed field '" + name + "' must have index options");
}
- if (omitNorms) {
- if (normType != null) {
- throw new IllegalStateException("indexed field '" + name + "' cannot both omit norms and have norms");
- }
- }
// Cannot store payloads unless positions are indexed:
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0 && storePayloads) {
throw new IllegalStateException("indexed field '" + name + "' cannot have payloads without positions");
@@ -177,12 +170,8 @@ public final class FieldInfo {
if (omitNorms) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot omit norms");
}
- if (normType != null) {
- throw new IllegalStateException("non-indexed field '" + name + "' cannot have norms");
- }
if (indexOptions != null) {
throw new IllegalStateException("non-indexed field '" + name + "' cannot have index options");
-
}
}
@@ -206,7 +195,6 @@ public final class FieldInfo {
this.storePayloads |= storePayloads;
if (this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
- this.normType = null;
}
if (this.indexOptions != indexOptions) {
if (this.indexOptions == null) {
@@ -265,13 +253,6 @@ public final class FieldInfo {
return dvGen;
}
- /**
- * Returns {@link DocValuesType} of the norm. this may be null if the field has no norms.
- */
- public DocValuesType getNormType() {
- return normType;
- }
-
void setStoreTermVectors() {
storeTermVector = true;
assert checkConsistency();
@@ -284,14 +265,6 @@ public final class FieldInfo {
assert checkConsistency();
}
- void setNormValueType(DocValuesType type) {
- if (normType != null && normType != type) {
- throw new IllegalArgumentException("cannot change Norm type from " + normType + " to " + type + " for field \"" + name + "\"");
- }
- normType = type;
- assert checkConsistency();
- }
-
/**
* Returns true if norms are explicitly omitted for this field
*/
@@ -303,7 +276,7 @@ public final class FieldInfo {
* Returns true if this field actually has any norms.
*/
public boolean hasNorms() {
- return normType != null;
+ return indexed && omitNorms == false;
}
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Wed Oct 15 16:58:30 2014
@@ -276,10 +276,8 @@ public class FieldInfos implements Itera
}
/** NOTE: this method does not carry over termVector
- * booleans nor docValuesType; the indexer chain
- * (TermVectorsConsumerPerField, DocFieldProcessor) must
- * set these fields when they succeed in consuming
- * the document */
+ * the indexer chain must set these fields when they
+ * succeed in consuming the document */
public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
// TODO: really, indexer shouldn't even call this
// method (it's only called from DocFieldProcessor);
@@ -288,12 +286,12 @@ public class FieldInfos implements Itera
// be updated by maybe FreqProxTermsWriterPerField:
return addOrUpdateInternal(name, -1, fieldType.indexed(), false,
fieldType.omitNorms(), false,
- fieldType.indexOptions(), fieldType.docValueType(), null);
+ fieldType.indexOptions(), fieldType.docValueType());
}
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed,
boolean storeTermVector,
- boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues, DocValuesType normType) {
+ boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
FieldInfo fi = fieldInfo(name);
if (fi == null) {
// This field wasn't yet added to this in-RAM
@@ -302,7 +300,7 @@ public class FieldInfos implements Itera
// before then we'll get the same name and number,
// else we'll allocate a new one:
final int fieldNumber = globalFieldNumbers.addOrGet(name, preferredFieldNumber, docValues);
- fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, normType, -1, null);
+ fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, -1, null);
assert !byName.containsKey(fi.name);
assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType());
byName.put(fi.name, fi);
@@ -319,26 +317,22 @@ public class FieldInfos implements Itera
globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
}
}
-
- if (!fi.omitsNorms() && normType != null) {
- fi.setNormValueType(normType);
- }
}
return fi;
}
-
+
public FieldInfo add(FieldInfo fi) {
// IMPORTANT - reuse the field number if possible for consistent field numbers across segments
return addOrUpdateInternal(fi.name, fi.number, fi.isIndexed(), fi.hasVectors(),
fi.omitsNorms(), fi.hasPayloads(),
- fi.getIndexOptions(), fi.getDocValuesType(), fi.getNormType());
+ fi.getIndexOptions(), fi.getDocValuesType());
}
public FieldInfo fieldInfo(String fieldName) {
return byName.get(fieldName);
}
- final FieldInfos finish() {
+ FieldInfos finish() {
return new FieldInfos(byName.values().toArray(new FieldInfo[byName.size()]));
}
}
Modified: lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/trunk/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Wed Oct 15 16:58:30 2014
@@ -445,7 +445,7 @@ public class MemoryIndex {
if (!fieldInfos.containsKey(fieldName)) {
fieldInfos.put(fieldName,
- new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, null, -1, null));
+ new FieldInfo(fieldName, true, fieldInfos.size(), false, false, false, this.storeOffsets ? IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS , null, -1, null));
}
TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);
PositionIncrementAttribute posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java Wed Oct 15 16:58:30 2014
@@ -21,22 +21,22 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Map;
+import org.apache.lucene.document.BinaryDocValuesField; // javadocs
+import org.apache.lucene.document.DoubleField; // javadocs
+import org.apache.lucene.document.FloatField; // javadocs
import org.apache.lucene.document.IntField; // javadocs
import org.apache.lucene.document.LongField; // javadocs
-import org.apache.lucene.document.FloatField; // javadocs
-import org.apache.lucene.document.DoubleField; // javadocs
-import org.apache.lucene.document.BinaryDocValuesField; // javadocs
import org.apache.lucene.document.NumericDocValuesField; // javadocs
import org.apache.lucene.document.SortedDocValuesField; // javadocs
import org.apache.lucene.document.SortedSetDocValuesField; // javadocs
import org.apache.lucene.document.StringField; // javadocs
-import org.apache.lucene.index.FilterLeafReader;
-import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FilterDirectoryReader;
+import org.apache.lucene.index.FilterLeafReader;
+import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
@@ -213,7 +213,7 @@ public class UninvertingReader extends F
}
}
filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
- fi.hasPayloads(), fi.getIndexOptions(), type, fi.getNormType(), -1, null));
+ fi.hasPayloads(), fi.getIndexOptions(), type, -1, null));
}
fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingNormsFormat.java Wed Oct 15 16:58:30 2014
@@ -93,7 +93,7 @@ public class AssertingNormsFormat extend
@Override
public NumericDocValues getNorms(FieldInfo field) throws IOException {
- assert field.getNormType() == FieldInfo.DocValuesType.NUMERIC;
+ assert field.hasNorms();
NumericDocValues values = in.getNorms(field);
assert values != null;
return new AssertingLeafReader.AssertingNumericDocValues(values, maxDoc);
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Wed Oct 15 16:58:30 2014
@@ -89,11 +89,6 @@ public abstract class BaseFieldInfoForma
fi.setStorePayloads();
}
}
- if (fi.isIndexed() && !fi.omitsNorms()) {
- if (random().nextBoolean()) {
- fi.setNormValueType(DocValuesType.NUMERIC);
- }
- }
addAttributes(fi);
}
FieldInfos infos = builder.finish();
@@ -165,7 +160,6 @@ public abstract class BaseFieldInfoForma
assertEquals(expected.name, actual.name);
assertEquals(expected.getDocValuesType(), actual.getDocValuesType());
assertEquals(expected.getIndexOptions(), actual.getIndexOptions());
- assertEquals(expected.getNormType(), actual.getNormType());
assertEquals(expected.hasDocValues(), actual.hasDocValues());
assertEquals(expected.hasNorms(), actual.hasNorms());
assertEquals(expected.hasPayloads(), actual.hasPayloads());
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BaseNormsFormatTestCase.java Wed Oct 15 16:58:30 2014
@@ -18,6 +18,8 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@@ -128,7 +130,6 @@ public abstract class BaseNormsFormatTes
public void testAllZeros() throws Exception {
int iterations = atLeast(1);
- final Random r = random();
for (int i = 0; i < iterations; i++) {
doTestNormsVersusStoredFields(new LongProducer() {
@Override
@@ -264,4 +265,60 @@ public abstract class BaseNormsFormatTes
}
// TODO: test thread safety (e.g. across different fields) explicitly here
+
+ /**
+ * LUCENE-6006: Tests undead norms.
+ * .....
+ * C C /
+ * /< /
+ * ___ __________/_#__=o
+ * /(- /(\_\________ \
+ * \ ) \ )_ \o \
+ * /|\ /|\ |' |
+ * | _|
+ * /o __\
+ * / ' |
+ * / / |
+ * /_/\______|
+ * ( _( <
+ * \ \ \
+ * \ \ |
+ * \____\____\
+ * ____\_\__\_\
+ * /` /` o\
+ * |___ |_______|
+ *
+ */
+ public void testUndeadNorms() throws Exception {
+ Directory dir = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random(), dir);
+ int numDocs = atLeast(1000);
+ List<Integer> toDelete = new ArrayList<>();
+ for(int i=0;i<numDocs;i++) {
+ Document doc = new Document();
+ doc.add(new StringField("id", ""+i, Field.Store.NO));
+ if (random().nextInt(5) == 1) {
+ toDelete.add(i);
+ doc.add(new TextField("content", "some content", Field.Store.NO));
+ }
+ w.addDocument(doc);
+ }
+ for(Integer id : toDelete) {
+ w.deleteDocuments(new Term("id", ""+id));
+ }
+ w.forceMerge(1);
+ IndexReader r = w.getReader();
+
+ // Confusingly, norms should exist, and should all be 0, even though we deleted all docs that had the field "content". They should not
+ // be undead:
+ NumericDocValues norms = MultiDocValues.getNormValues(r, "content");
+ assertNotNull(norms);
+ for(int i=0;i<r.maxDoc();i++) {
+ assertEquals(0, norms.get(i));
+ }
+
+ r.close();
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java Wed Oct 15 16:58:30 2014
@@ -45,7 +45,6 @@ import org.apache.lucene.codecs.perfield
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.Directory;
@@ -373,7 +372,7 @@ public abstract class BasePostingsFormat
fieldInfoArray[fieldUpto] = new FieldInfo(field, true, fieldUpto, false, false, true,
IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
- null, DocValuesType.NUMERIC, -1, null);
+ null, -1, null);
fieldUpto++;
SortedMap<BytesRef,SeedAndOrd> postings = new TreeMap<>();
@@ -702,7 +701,6 @@ public abstract class BasePostingsFormat
doPayloads,
indexOptions,
null,
- DocValuesType.NUMERIC,
-1,
null);
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/Insanity.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/Insanity.java?rev=1632120&r1=1632119&r2=1632120&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/Insanity.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/Insanity.java Wed Oct 15 16:58:30 2014
@@ -65,7 +65,7 @@ public class Insanity {
for (FieldInfo fi : in.getFieldInfos()) {
if (fi.name.equals(insaneField)) {
filteredInfos.add(new FieldInfo(fi.name, fi.isIndexed(), fi.number, fi.hasVectors(), fi.omitsNorms(),
- fi.hasPayloads(), fi.getIndexOptions(), null, fi.getNormType(), -1, null));
+ fi.hasPayloads(), fi.getIndexOptions(), null, -1, null));
} else {
filteredInfos.add(fi);
}