You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/08 09:51:36 UTC
svn commit: r1637528 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/document/
lucene/core/src/java/org/apache/lucene/index/
lucene/core/src/test/org/apache/lucene/document/
lucene/core/src/test/org/a...
Author: mikemccand
Date: Sat Nov 8 08:51:35 2014
New Revision: 1637528
URL: http://svn.apache.org/r1637528
Log:
LUCENE-6049: don't throw exc writing segment if document hit non-aborting exc
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
lucene/dev/branches/branch_5x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/ (props changed)
lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java
lucene/dev/branches/branch_5x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/BBoxField.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Sat Nov 8 08:51:35 2014
@@ -209,6 +209,10 @@ Bug Fixes
* LUCENE-6054: Allow repeating the empty automaton (Nik Everett via
Mike McCandless)
+* LUCENE-6049: Don't throw cryptic exception writing a segment when
+ the only docs in it had fields that hit non-aborting exceptions
+ during indexing but also had doc values. (Mike McCandless)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/BinaryDocValuesField.java Sat Nov 8 08:51:35 2014
@@ -46,7 +46,7 @@ public class BinaryDocValuesField extend
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.BINARY);
+ TYPE.setDocValuesType(DocValuesType.BINARY);
TYPE.freeze();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/FieldType.java Sat Nov 8 08:51:35 2014
@@ -54,7 +54,7 @@ public class FieldType implements Indexa
private NumericType numericType;
private boolean frozen;
private int numericPrecisionStep = NumericUtils.PRECISION_STEP_DEFAULT;
- private DocValuesType docValueType = DocValuesType.NONE;
+ private DocValuesType docValuesType = DocValuesType.NONE;
/**
* Create a new mutable FieldType with all of the properties from <code>ref</code>
@@ -68,7 +68,7 @@ public class FieldType implements Indexa
this.storeTermVectorPayloads = ref.storeTermVectorPayloads();
this.omitNorms = ref.omitNorms();
this.indexOptions = ref.indexOptions();
- this.docValueType = ref.docValueType();
+ this.docValuesType = ref.docValuesType();
this.numericType = ref.numericType();
// Do not copy frozen!
}
@@ -377,12 +377,12 @@ public class FieldType implements Indexa
result.append(numericPrecisionStep);
}
}
- if (docValueType != DocValuesType.NONE) {
+ if (docValuesType != DocValuesType.NONE) {
if (result.length() > 0) {
result.append(",");
}
- result.append("docValueType=");
- result.append(docValueType);
+ result.append("docValuesType=");
+ result.append(docValuesType);
}
return result.toString();
@@ -392,11 +392,11 @@ public class FieldType implements Indexa
* {@inheritDoc}
* <p>
* The default is <code>null</code> (no docValues)
- * @see #setDocValueType(DocValuesType)
+ * @see #setDocValuesType(DocValuesType)
*/
@Override
- public DocValuesType docValueType() {
- return docValueType;
+ public DocValuesType docValuesType() {
+ return docValuesType;
}
/**
@@ -404,21 +404,21 @@ public class FieldType implements Indexa
* @param type DocValues type, or null if no DocValues should be stored.
* @throws IllegalStateException if this FieldType is frozen against
* future modifications.
- * @see #docValueType()
+ * @see #docValuesType()
*/
- public void setDocValueType(DocValuesType type) {
+ public void setDocValuesType(DocValuesType type) {
checkIfFrozen();
if (type == null) {
throw new NullPointerException("DocValuesType cannot be null");
}
- docValueType = type;
+ docValuesType = type;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
- result = prime * result + ((docValueType == null) ? 0 : docValueType.hashCode());
+ result = prime * result + ((docValuesType == null) ? 0 : docValuesType.hashCode());
result = prime * result + indexOptions.hashCode();
result = prime * result + numericPrecisionStep;
result = prime * result + ((numericType == null) ? 0 : numericType.hashCode());
@@ -438,7 +438,7 @@ public class FieldType implements Indexa
if (obj == null) return false;
if (getClass() != obj.getClass()) return false;
FieldType other = (FieldType) obj;
- if (docValueType != other.docValueType) return false;
+ if (docValuesType != other.docValuesType) return false;
if (indexOptions != other.indexOptions) return false;
if (numericPrecisionStep != other.numericPrecisionStep) return false;
if (numericType != other.numericType) return false;
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java Sat Nov 8 08:51:35 2014
@@ -40,7 +40,7 @@ public class NumericDocValuesField exten
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.NUMERIC);
+ TYPE.setDocValuesType(DocValuesType.NUMERIC);
TYPE.freeze();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java Sat Nov 8 08:51:35 2014
@@ -43,7 +43,7 @@ public class SortedDocValuesField extend
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.SORTED);
+ TYPE.setDocValuesType(DocValuesType.SORTED);
TYPE.freeze();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java Sat Nov 8 08:51:35 2014
@@ -50,7 +50,7 @@ public class SortedNumericDocValuesField
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.SORTED_NUMERIC);
+ TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC);
TYPE.freeze();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java Sat Nov 8 08:51:35 2014
@@ -44,7 +44,7 @@ public class SortedSetDocValuesField ext
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(DocValuesType.SORTED_SET);
+ TYPE.setDocValuesType(DocValuesType.SORTED_SET);
TYPE.freeze();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Nov 8 08:51:35 2014
@@ -381,9 +381,9 @@ final class DefaultIndexingChain extends
}
}
- DocValuesType dvType = fieldType.docValueType();
+ DocValuesType dvType = fieldType.docValuesType();
if (dvType == null) {
- throw new NullPointerException("docValueType cannot be null (field: \"" + fieldName + "\")");
+ throw new NullPointerException("docValuesType cannot be null (field: \"" + fieldName + "\")");
}
if (dvType != DocValuesType.NONE) {
if (fp == null) {
@@ -496,7 +496,11 @@ final class DefaultIndexingChain extends
if (fp == null) {
// First time we are seeing this field in this segment
- FieldInfo fi = fieldInfos.addOrUpdate(name, fieldType);
+ FieldInfo fi = fieldInfos.getOrAdd(name);
+ // Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
+ // IndexOptions to decide what arrays it must create). Then, we also must set it in
+ // PerField.invert to allow for later downgrading of the index options:
+ fi.setIndexOptions(fieldType.indexOptions());
fp = new PerField(fi, invert);
fp.next = fieldHash[hashPos];
@@ -514,12 +518,12 @@ final class DefaultIndexingChain extends
fields = newFields;
}
- } else {
- fp.fieldInfo.update(fieldType);
-
- if (invert && fp.invertState == null) {
- fp.setInvertState();
- }
+ } else if (invert && fp.invertState == null) {
+ // Messy: must set this here because e.g. FreqProxTermsWriterPerField looks at the initial
+ // IndexOptions to decide what arrays it must create). Then, we also must set it in
+ // PerField.invert to allow for later downgrading of the index options:
+ fp.fieldInfo.setIndexOptions(fieldType.indexOptions());
+ fp.setInvertState();
}
return fp;
@@ -551,6 +555,8 @@ final class DefaultIndexingChain extends
// reused
TokenStream tokenStream;
+ IndexOptions indexOptions;
+
public PerField(FieldInfo fieldInfo, boolean invert) {
this.fieldInfo = fieldInfo;
similarity = docState.similarity;
@@ -594,11 +600,18 @@ final class DefaultIndexingChain extends
IndexableFieldType fieldType = field.fieldType();
+ IndexOptions indexOptions = fieldType.indexOptions();
+ fieldInfo.setIndexOptions(indexOptions);
+
+ if (fieldType.omitNorms()) {
+ fieldInfo.setOmitsNorms();
+ }
+
final boolean analyzed = fieldType.tokenized() && docState.analyzer != null;
// only bother checking offsets if something will consume them.
// TODO: after we fix analyzers, also check if termVectorOffsets will be indexed.
- final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+ final boolean checkOffsets = indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
/*
* To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Sat Nov 8 08:51:35 2014
@@ -108,10 +108,6 @@ public final class FieldInfo {
return true;
}
- void update(IndexableFieldType ft) {
- update(false, ft.omitNorms(), false, ft.indexOptions());
- }
-
// should only be called by FieldInfos#addOrUpdate
void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
if (indexOptions == null) {
@@ -144,7 +140,7 @@ public final class FieldInfo {
}
void setDocValuesType(DocValuesType type) {
- if (docValuesType != DocValuesType.NONE && docValuesType != type) {
+ if (docValuesType != DocValuesType.NONE && type != DocValuesType.NONE && docValuesType != type) {
throw new IllegalArgumentException("cannot change DocValues type from " + docValuesType + " to " + type + " for field \"" + name + "\"");
}
docValuesType = type;
@@ -155,6 +151,23 @@ public final class FieldInfo {
public IndexOptions getIndexOptions() {
return indexOptions;
}
+
+ /** Record the {@link IndexOptions} to use with this field. */
+ public void setIndexOptions(IndexOptions newIndexOptions) {
+ if (indexOptions != newIndexOptions) {
+ if (indexOptions == IndexOptions.NONE) {
+ indexOptions = newIndexOptions;
+ } else if (newIndexOptions != IndexOptions.NONE) {
+ // downgrade
+ indexOptions = indexOptions.compareTo(newIndexOptions) < 0 ? indexOptions : newIndexOptions;
+ }
+ }
+
+ if (indexOptions == IndexOptions.NONE || indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+ // cannot store payloads if we don't store positions:
+ storePayloads = false;
+ }
+ }
/**
* Returns {@link DocValuesType} of the docValues; this is
@@ -196,6 +209,14 @@ public final class FieldInfo {
public boolean omitsNorms() {
return omitNorms;
}
+
+ /** Omit norms for this field. */
+ public void setOmitsNorms() {
+ if (indexOptions == IndexOptions.NONE) {
+ throw new IllegalStateException("cannot omit norms: this field is not indexed");
+ }
+ omitNorms = true;
+ }
/**
* Returns true if this field actually has any norms.
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Sat Nov 8 08:51:35 2014
@@ -276,21 +276,26 @@ public class FieldInfos implements Itera
add(fieldInfo);
}
}
-
- /** NOTE: this method does not carry over termVector
- * the indexer chain must set these fields when they
- * succeed in consuming the document */
- public FieldInfo addOrUpdate(String name, IndexableFieldType fieldType) {
- // TODO: really, indexer shouldn't even call this
- // method (it's only called from DocFieldProcessor);
- // rather, each component in the chain should update
- // what it "owns". EG fieldType.indexOptions() should
- // be updated by maybe FreqProxTermsWriterPerField:
- return addOrUpdateInternal(name, -1, false,
- fieldType.omitNorms(), false,
- fieldType.indexOptions(), fieldType.docValueType());
- }
+ /** Create a new field, or return existing one. */
+ public FieldInfo getOrAdd(String name) {
+ FieldInfo fi = fieldInfo(name);
+ if (fi == null) {
+ // This field wasn't yet added to this in-RAM
+ // segment's FieldInfo, so now we get a global
+ // number for this field. If the field was seen
+ // before then we'll get the same name and number,
+ // else we'll allocate a new one:
+ final int fieldNumber = globalFieldNumbers.addOrGet(name, -1, DocValuesType.NONE);
+ fi = new FieldInfo(name, fieldNumber, false, false, false, IndexOptions.NONE, DocValuesType.NONE, -1, null);
+ assert !byName.containsKey(fi.name);
+ globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, DocValuesType.NONE);
+ byName.put(fi.name, fi);
+ }
+
+ return fi;
+ }
+
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
@@ -317,7 +322,7 @@ public class FieldInfos implements Itera
boolean updateGlobal = fi.getDocValuesType() == DocValuesType.NONE;
if (updateGlobal) {
// Must also update docValuesType map so it's
- // aware of this field's DocValueType. This will throw IllegalArgumentException if
+ // aware of this field's DocValuesType. This will throw IllegalArgumentException if
// an illegal type change was attempted.
globalFieldNumbers.setDocValuesType(fi.number, name, docValues);
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Nov 8 08:51:35 2014
@@ -1496,7 +1496,7 @@ public class IndexWriter implements Clos
DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length];
for (int i = 0; i < updates.length; i++) {
final Field f = updates[i];
- final DocValuesType dvType = f.fieldType().docValueType();
+ final DocValuesType dvType = f.fieldType().docValuesType();
if (dvType == null) {
throw new NullPointerException("DocValuesType cannot be null (field: \"" + f.name() + "\")");
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Sat Nov 8 08:51:35 2014
@@ -95,5 +95,5 @@ public interface IndexableFieldType {
* DocValues {@link DocValuesType}: how the field's value will be indexed
* into docValues.
*/
- public DocValuesType docValueType();
+ public DocValuesType docValuesType();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/ReadersAndUpdates.java Sat Nov 8 08:51:35 2014
@@ -496,11 +496,13 @@ class ReadersAndUpdates {
}
// create new fields or update existing ones to have NumericDV type
for (String f : dvUpdates.numericDVUpdates.keySet()) {
- builder.addOrUpdate(f, NumericDocValuesField.TYPE);
+ FieldInfo fieldInfo = builder.getOrAdd(f);
+ fieldInfo.setDocValuesType(DocValuesType.NUMERIC);
}
// create new fields or update existing ones to have BinaryDV type
for (String f : dvUpdates.binaryDVUpdates.keySet()) {
- builder.addOrUpdate(f, BinaryDocValuesField.TYPE);
+ FieldInfo fieldInfo = builder.getOrAdd(f);
+ fieldInfo.setDocValuesType(DocValuesType.BINARY);
}
fieldInfos = builder.finish();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Sat Nov 8 08:51:35 2014
@@ -86,7 +86,7 @@ abstract class TermsHash {
}
}
- abstract TermsHashPerField addField(FieldInvertState fieldInvertState, final FieldInfo fieldInfo);
+ abstract TermsHashPerField addField(FieldInvertState fieldInvertState, FieldInfo fieldInfo);
void finishDocument() throws IOException {
if (nextTermsHash != null) {
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java Sat Nov 8 08:51:35 2014
@@ -39,7 +39,7 @@ public class TestFieldType extends Lucen
assertFalse(ft3.equals(ft));
FieldType ft4 = new FieldType();
- ft4.setDocValueType(DocValuesType.BINARY);
+ ft4.setDocValuesType(DocValuesType.BINARY);
assertFalse(ft4.equals(ft));
FieldType ft5 = new FieldType();
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestBinaryDocValuesUpdates.java Sat Nov 8 08:51:35 2014
@@ -833,7 +833,7 @@ public class TestBinaryDocValuesUpdates
public void testUpdateBinaryDVFieldWithSameNameAsPostingField() throws Exception {
// this used to fail because FieldInfos.Builder neglected to update
- // globalFieldMaps.docValueTypes map
+ // globalFieldMaps.docValuesTypes map
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestCodecs.java Sat Nov 8 08:51:35 2014
@@ -88,35 +88,12 @@ public class TestCodecs extends LuceneTe
this.omitTF = omitTF;
this.storePayloads = storePayloads;
// TODO: change this test to use all three
- fieldInfo = fieldInfos.addOrUpdate(name, new IndexableFieldType() {
-
- @Override
- public boolean stored() { return false; }
-
- @Override
- public boolean tokenized() { return false; }
-
- @Override
- public boolean storeTermVectors() { return false; }
-
- @Override
- public boolean storeTermVectorOffsets() { return false; }
-
- @Override
- public boolean storeTermVectorPositions() { return false; }
-
- @Override
- public boolean storeTermVectorPayloads() { return false; }
-
- @Override
- public boolean omitNorms() { return false; }
-
- @Override
- public IndexOptions indexOptions() { return omitTF ? IndexOptions.DOCS : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; }
-
- @Override
- public DocValuesType docValueType() { return DocValuesType.NONE; }
- });
+ fieldInfo = fieldInfos.getOrAdd(name);
+ if (omitTF) {
+ fieldInfo.setIndexOptions(IndexOptions.DOCS);
+ } else {
+ fieldInfo.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
+ }
if (storePayloads) {
fieldInfo.setStorePayloads();
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Sat Nov 8 08:51:35 2014
@@ -23,10 +23,12 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.SortedSetDocValuesField;
@@ -898,4 +900,32 @@ public class TestDocValuesIndexing exten
dir.close();
}
+
+ // LUCENE-6049
+ public void testExcIndexingDocBeforeDocValues() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
+ IndexWriter w = new IndexWriter(dir, iwc);
+ Document doc = new Document();
+ FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
+ ft.setDocValuesType(DocValuesType.SORTED);
+ ft.freeze();
+ Field field = new Field("test", "value", ft);
+ field.setTokenStream(new TokenStream() {
+ @Override
+ public boolean incrementToken() {
+ throw new RuntimeException("no");
+ }
+ });
+ doc.add(field);
+ try {
+ w.addDocument(doc);
+ fail("did not hit exception");
+ } catch (RuntimeException re) {
+ // expected
+ }
+ w.addDocument(new Document());
+ w.close();
+ dir.close();
+ }
}
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java Sat Nov 8 08:51:35 2014
@@ -50,7 +50,13 @@ public class TestFieldsReader extends Lu
fieldInfos = new FieldInfos.Builder();
DocHelper.setupDoc(testDoc);
for (IndexableField field : testDoc.getFields()) {
- fieldInfos.addOrUpdate(field.name(), field.fieldType());
+ FieldInfo fieldInfo = fieldInfos.getOrAdd(field.name());
+ IndexableFieldType ift = field.fieldType();
+ fieldInfo.setIndexOptions(ift.indexOptions());
+ if (ift.omitNorms()) {
+ fieldInfo.setOmitsNorms();
+ }
+ fieldInfo.setDocValuesType(ift.docValuesType());
}
dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()))
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Sat Nov 8 08:51:35 2014
@@ -87,7 +87,7 @@ public class TestIndexableField extends
}
@Override
- public DocValuesType docValueType() {
+ public DocValuesType docValuesType() {
return DocValuesType.NONE;
}
};
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestNumericDocValuesUpdates.java Sat Nov 8 08:51:35 2014
@@ -826,7 +826,7 @@ public class TestNumericDocValuesUpdates
@Test
public void testUpdateNumericDVFieldWithSameNameAsPostingField() throws Exception {
// this used to fail because FieldInfos.Builder neglected to update
- // globalFieldMaps.docValueTypes map
+ // globalFieldMaps.docValuesTypes map
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
IndexWriter writer = new IndexWriter(dir, conf);
Modified: lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Sat Nov 8 08:51:35 2014
@@ -106,7 +106,7 @@ public class BBoxStrategy extends Spatia
FieldType fieldType = new FieldType(DoubleField.TYPE_NOT_STORED);
fieldType.setNumericPrecisionStep(8);//Solr's default
- fieldType.setDocValueType(DocValuesType.NUMERIC);
+ fieldType.setDocValuesType(DocValuesType.NUMERIC);
setFieldType(fieldType);
}
Modified: lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java (original)
+++ lucene/dev/branches/branch_5x/lucene/spatial/src/test/org/apache/lucene/spatial/bbox/TestBBoxStrategy.java Sat Nov 8 08:51:35 2014
@@ -112,7 +112,7 @@ public class TestBBoxStrategy extends Ra
if (random().nextBoolean()) {
BBoxStrategy bboxStrategy = (BBoxStrategy) strategy;
FieldType fieldType = new FieldType(bboxStrategy.getFieldType());
- fieldType.setDocValueType(DocValuesType.NONE);
+ fieldType.setDocValuesType(DocValuesType.NONE);
bboxStrategy.setFieldType(fieldType);
}
for (SpatialOperation operation : SpatialOperation.values()) {
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/index/BaseFieldInfoFormatTestCase.java Sat Nov 8 08:51:35 2014
@@ -49,7 +49,8 @@ public abstract class BaseFieldInfoForma
Codec codec = getCodec();
SegmentInfo segmentInfo = newSegmentInfo(dir, "_123");
FieldInfos.Builder builder = new FieldInfos.Builder();
- FieldInfo fi = builder.addOrUpdate("field", TextField.TYPE_STORED);
+ FieldInfo fi = builder.getOrAdd("field");
+ fi.setIndexOptions(TextField.TYPE_STORED.indexOptions());
addAttributes(fi);
FieldInfos infos = builder.finish();
codec.fieldInfosFormat().write(dir, segmentInfo, "", infos, IOContext.DEFAULT);
@@ -81,7 +82,15 @@ public abstract class BaseFieldInfoForma
FieldInfos.Builder builder = new FieldInfos.Builder();
for (String field : fieldNames) {
IndexableFieldType fieldType = randomFieldType(random());
- FieldInfo fi = builder.addOrUpdate(field, fieldType);
+ FieldInfo fi = builder.getOrAdd(field);
+ IndexOptions indexOptions = fieldType.indexOptions();
+ if (indexOptions != IndexOptions.NONE) {
+ fi.setIndexOptions(indexOptions);
+ if (fieldType.omitNorms()) {
+ fi.setOmitsNorms();
+ }
+ }
+ fi.setDocValuesType(fieldType.docValuesType());
if (fieldType.indexOptions() != IndexOptions.NONE && fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
if (random().nextBoolean()) {
fi.setStorePayloads();
@@ -118,7 +127,7 @@ public abstract class BaseFieldInfoForma
if (r.nextBoolean()) {
DocValuesType values[] = getDocValuesTypes();
- type.setDocValueType(values[r.nextInt(values.length)]);
+ type.setDocValuesType(values[r.nextInt(values.length)]);
}
return type;
Modified: lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java (original)
+++ lucene/dev/branches/branch_5x/lucene/test-framework/src/java/org/apache/lucene/util/TestUtil.java Sat Nov 8 08:51:35 2014
@@ -887,7 +887,7 @@ public final class TestUtil {
for(IndexableField f : doc1.getFields()) {
final Field field1 = (Field) f;
final Field field2;
- final DocValuesType dvType = field1.fieldType().docValueType();
+ final DocValuesType dvType = field1.fieldType().docValuesType();
final NumericType numType = field1.fieldType().numericType();
if (dvType != DocValuesType.NONE) {
switch(dvType) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/LukeRequestHandler.java Sat Nov 8 08:51:35 2014
@@ -181,7 +181,7 @@ public class LukeRequestHandler extends
flags.append( (f != null && f.fieldType().indexOptions() != IndexOptions.NONE) ? FieldFlag.INDEXED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().tokenized()) ? FieldFlag.TOKENIZED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().stored()) ? FieldFlag.STORED.getAbbreviation() : '-' );
- flags.append( (f != null && f.fieldType().docValueType() != DocValuesType.NONE) ? FieldFlag.DOC_VALUES.getAbbreviation() : "-" );
+ flags.append( (f != null && f.fieldType().docValuesType() != DocValuesType.NONE) ? FieldFlag.DOC_VALUES.getAbbreviation() : "-" );
flags.append( (false) ? FieldFlag.MULTI_VALUED.getAbbreviation() : '-' ); // SchemaField Specific
flags.append( (f != null && f.fieldType().storeTermVectors()) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-' );
flags.append( (f != null && f.fieldType().storeTermVectorOffsets()) ? FieldFlag.TERM_VECTOR_OFFSET.getAbbreviation() : '-' );
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/BBoxField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/BBoxField.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/BBoxField.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/BBoxField.java Sat Nov 8 08:51:35 2014
@@ -103,7 +103,7 @@ public class BBoxField extends AbstractS
//and annoyingly this field isn't going to have a docValues format because Solr uses a separate Field for that
if (field.hasDocValues()) {
luceneType = new org.apache.lucene.document.FieldType(luceneType);
- luceneType.setDocValueType(DocValuesType.NUMERIC);
+ luceneType.setDocValuesType(DocValuesType.NUMERIC);
}
strategy.setFieldType(luceneType);
return strategy;
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java?rev=1637528&r1=1637527&r2=1637528&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/schema/FieldType.java Sat Nov 8 08:51:35 2014
@@ -303,7 +303,7 @@ public abstract class FieldType extends
*/
public List<IndexableField> createFields(SchemaField field, Object value, float boost) {
IndexableField f = createField( field, value, boost);
- if (field.hasDocValues() && f.fieldType().docValueType() == null) {
+ if (field.hasDocValues() && f.fieldType().docValuesType() == null) {
// field types that support doc values should either override createField
// to return a field with doc values or extend createFields if this can't
// be done in a single field instance (see StrField for example)