You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/11/01 10:32:12 UTC
svn commit: r1635908 [3/7] - in /lucene/dev/branches/lucene6005: ./
dev-tools/ dev-tools/idea/.idea/libraries/ dev-tools/scripts/ lucene/
lucene/backward-codecs/
lucene/backward-codecs/src/test/org/apache/lucene/index/ lucene/codecs/
lucene/codecs/src/...
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/IntField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/IntField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/IntField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/IntField.java Sat Nov 1 09:32:06 2014
@@ -18,7 +18,7 @@ package org.apache.lucene.document;
*/
import org.apache.lucene.analysis.NumericTokenStream; // javadocs
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.search.NumericRangeFilter; // javadocs
import org.apache.lucene.search.NumericRangeQuery; // javadocs
import org.apache.lucene.util.NumericUtils;
@@ -121,7 +121,7 @@ public final class IntField extends Fiel
static {
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.setOmitNorms(true);
- TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS);
TYPE_NOT_STORED.setNumericType(FieldType.NumericType.INT);
TYPE_NOT_STORED.setNumericPrecisionStep(NumericUtils.PRECISION_STEP_DEFAULT_32);
TYPE_NOT_STORED.freeze();
@@ -135,7 +135,7 @@ public final class IntField extends Fiel
static {
TYPE_STORED.setTokenized(true);
TYPE_STORED.setOmitNorms(true);
- TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_STORED.setIndexOptions(IndexOptions.DOCS);
TYPE_STORED.setNumericType(FieldType.NumericType.INT);
TYPE_STORED.setNumericPrecisionStep(NumericUtils.PRECISION_STEP_DEFAULT_32);
TYPE_STORED.setStored(true);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/LongField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/LongField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/LongField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/LongField.java Sat Nov 1 09:32:06 2014
@@ -18,7 +18,7 @@ package org.apache.lucene.document;
*/
import org.apache.lucene.analysis.NumericTokenStream; // javadocs
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.search.NumericRangeFilter; // javadocs
import org.apache.lucene.search.NumericRangeQuery; // javadocs
import org.apache.lucene.util.NumericUtils;
@@ -131,7 +131,7 @@ public final class LongField extends Fie
static {
TYPE_NOT_STORED.setTokenized(true);
TYPE_NOT_STORED.setOmitNorms(true);
- TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS);
TYPE_NOT_STORED.setNumericType(FieldType.NumericType.LONG);
TYPE_NOT_STORED.freeze();
}
@@ -144,7 +144,7 @@ public final class LongField extends Fie
static {
TYPE_STORED.setTokenized(true);
TYPE_STORED.setOmitNorms(true);
- TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_STORED.setIndexOptions(IndexOptions.DOCS);
TYPE_STORED.setNumericType(FieldType.NumericType.LONG);
TYPE_STORED.setStored(true);
TYPE_STORED.freeze();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/NumericDocValuesField.java Sat Nov 1 09:32:06 2014
@@ -17,7 +17,7 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocValuesType;
/**
* <p>
@@ -40,7 +40,7 @@ public class NumericDocValuesField exten
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(FieldInfo.DocValuesType.NUMERIC);
+ TYPE.setDocValueType(DocValuesType.NUMERIC);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedDocValuesField.java Sat Nov 1 09:32:06 2014
@@ -17,7 +17,7 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.util.BytesRef;
/**
@@ -43,7 +43,7 @@ public class SortedDocValuesField extend
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(FieldInfo.DocValuesType.SORTED);
+ TYPE.setDocValueType(DocValuesType.SORTED);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedNumericDocValuesField.java Sat Nov 1 09:32:06 2014
@@ -17,7 +17,7 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.util.NumericUtils;
/**
@@ -50,7 +50,7 @@ public class SortedNumericDocValuesField
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(FieldInfo.DocValuesType.SORTED_NUMERIC);
+ TYPE.setDocValueType(DocValuesType.SORTED_NUMERIC);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/SortedSetDocValuesField.java Sat Nov 1 09:32:06 2014
@@ -17,7 +17,7 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.util.BytesRef;
/**
@@ -44,7 +44,7 @@ public class SortedSetDocValuesField ext
*/
public static final FieldType TYPE = new FieldType();
static {
- TYPE.setDocValueType(FieldInfo.DocValuesType.SORTED_SET);
+ TYPE.setDocValueType(DocValuesType.SORTED_SET);
TYPE.freeze();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StringField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StringField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StringField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/StringField.java Sat Nov 1 09:32:06 2014
@@ -17,7 +17,7 @@ package org.apache.lucene.document;
* limitations under the License.
*/
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexOptions;
/** A field that is indexed but not tokenized: the entire
* String value is indexed as a single token. For example
@@ -37,12 +37,12 @@ public final class StringField extends F
static {
TYPE_NOT_STORED.setOmitNorms(true);
- TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_NOT_STORED.setIndexOptions(IndexOptions.DOCS);
TYPE_NOT_STORED.setTokenized(false);
TYPE_NOT_STORED.freeze();
TYPE_STORED.setOmitNorms(true);
- TYPE_STORED.setIndexOptions(IndexOptions.DOCS_ONLY);
+ TYPE_STORED.setIndexOptions(IndexOptions.DOCS);
TYPE_STORED.setStored(true);
TYPE_STORED.setTokenized(false);
TYPE_STORED.freeze();
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/TextField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/TextField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/document/TextField.java Sat Nov 1 09:32:06 2014
@@ -20,7 +20,7 @@ package org.apache.lucene.document;
import java.io.Reader;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexOptions;
/** A field that is indexed and tokenized, without term
* vectors. For example this would be used on a 'body'
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesFieldUpdates.java Sat Nov 1 09:32:06 2014
@@ -1,15 +1,5 @@
package org.apache.lucene.index;
-import org.apache.lucene.document.BinaryDocValuesField;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.InPlaceMergeSorter;
-import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.PagedGrowableWriter;
-import org.apache.lucene.util.packed.PagedMutable;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -27,6 +17,15 @@ import org.apache.lucene.util.packed.Pag
* limitations under the License.
*/
+import org.apache.lucene.document.BinaryDocValuesField;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.InPlaceMergeSorter;
+import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PagedGrowableWriter;
+import org.apache.lucene.util.packed.PagedMutable;
+
/**
* A {@link DocValuesFieldUpdates} which holds updates of documents, of a single
* {@link BinaryDocValuesField}.
@@ -102,7 +101,7 @@ class BinaryDocValuesFieldUpdates extend
private final int bitsPerValue;
public BinaryDocValuesFieldUpdates(String field, int maxDoc) {
- super(field, FieldInfo.DocValuesType.BINARY);
+ super(field, DocValuesType.BINARY);
bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
offsets = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Nov 1 09:32:06 2014
@@ -37,7 +37,6 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.document.Document2;
import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Sat Nov 1 09:32:06 2014
@@ -29,8 +29,6 @@ import org.apache.lucene.codecs.NormsCon
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.document.FieldType;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.ArrayUtil;
@@ -341,8 +339,12 @@ final class DefaultIndexingChain extends
PerField fp = null;
+ if (fieldType.indexOptions() == null) {
+ throw new NullPointerException("IndexOptions must not be null (field: \"" + fieldName + "\")");
+ }
+
// Invert indexed fields:
- if (fieldType.indexOptions() != null) {
+ if (fieldType.indexOptions() != IndexOptions.NONE) {
// if the field omits norms, the boost cannot be indexed.
if (fieldType.omitNorms() && field.boost() != 1.0f) {
@@ -380,7 +382,10 @@ final class DefaultIndexingChain extends
}
DocValuesType dvType = fieldType.docValueType();
- if (dvType != null) {
+ if (dvType == null) {
+ throw new NullPointerException("docValueType cannot be null (field: \"" + fieldName + "\")");
+ }
+ if (dvType != DocValuesType.NONE) {
if (fp == null) {
fp = getOrAddField(fieldName, fieldType, false);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesFieldUpdates.java Sat Nov 1 09:32:06 2014
@@ -1,12 +1,5 @@
package org.apache.lucene.index;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.lucene.index.NumericDocValuesFieldUpdates;
-import org.apache.lucene.search.DocIdSetIterator;
-import org.apache.lucene.util.packed.PagedGrowableWriter;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -24,6 +17,12 @@ import org.apache.lucene.util.packed.Pag
* limitations under the License.
*/
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.packed.PagedGrowableWriter;
+
/**
* Holds updates of a single DocValues field, for a set of documents.
*
@@ -98,7 +97,7 @@ abstract class DocValuesFieldUpdates {
return ramBytesPerDoc;
}
- DocValuesFieldUpdates getUpdates(String field, FieldInfo.DocValuesType type) {
+ DocValuesFieldUpdates getUpdates(String field, DocValuesType type) {
switch (type) {
case NUMERIC:
return numericDVUpdates.get(field);
@@ -109,7 +108,7 @@ abstract class DocValuesFieldUpdates {
}
}
- DocValuesFieldUpdates newUpdates(String field, FieldInfo.DocValuesType type, int maxDoc) {
+ DocValuesFieldUpdates newUpdates(String field, DocValuesType type, int maxDoc) {
switch (type) {
case NUMERIC:
assert numericDVUpdates.get(field) == null;
@@ -133,10 +132,13 @@ abstract class DocValuesFieldUpdates {
}
final String field;
- final FieldInfo.DocValuesType type;
+ final DocValuesType type;
- protected DocValuesFieldUpdates(String field, FieldInfo.DocValuesType type) {
+ protected DocValuesFieldUpdates(String field, DocValuesType type) {
this.field = field;
+ if (type == null) {
+ throw new NullPointerException("DocValuesType cannot be null");
+ }
this.type = type;
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocValuesUpdate.java Sat Nov 1 09:32:06 2014
@@ -1,15 +1,5 @@
package org.apache.lucene.index;
-import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
-import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_CHAR;
-import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_INT;
-import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER;
-import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
-
-import org.apache.lucene.document.NumericDocValuesField;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.RamUsageEstimator;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -27,6 +17,16 @@ import org.apache.lucene.util.RamUsageEs
* limitations under the License.
*/
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_CHAR;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_INT;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_HEADER;
+import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.RamUsageEstimator;
+
/** An in-place update to a DocValues field. */
abstract class DocValuesUpdate {
@@ -39,7 +39,7 @@ abstract class DocValuesUpdate {
*/
private static final int RAW_SIZE_IN_BYTES = 8*NUM_BYTES_OBJECT_HEADER + 8*NUM_BYTES_OBJECT_REF + 8*NUM_BYTES_INT;
- final FieldInfo.DocValuesType type;
+ final DocValuesType type;
final Term term;
final String field;
final Object value;
@@ -52,7 +52,7 @@ abstract class DocValuesUpdate {
* @param field the {@link NumericDocValuesField} to update
* @param value the updated value
*/
- protected DocValuesUpdate(FieldInfo.DocValuesType type, Term term, String field, Object value) {
+ protected DocValuesUpdate(DocValuesType type, Term term, String field, Object value) {
this.type = type;
this.term = term;
this.field = field;
@@ -82,7 +82,7 @@ abstract class DocValuesUpdate {
private static final long RAW_VALUE_SIZE_IN_BYTES = NUM_BYTES_ARRAY_HEADER + 2*NUM_BYTES_INT + NUM_BYTES_OBJECT_REF;
BinaryDocValuesUpdate(Term term, String field, BytesRef value) {
- super(FieldInfo.DocValuesType.BINARY, term, field, value);
+ super(DocValuesType.BINARY, term, field, value);
}
@Override
@@ -96,7 +96,7 @@ abstract class DocValuesUpdate {
static final class NumericDocValuesUpdate extends DocValuesUpdate {
NumericDocValuesUpdate(Term term, String field, Long value) {
- super(FieldInfo.DocValuesType.NUMERIC, term, field, value);
+ super(DocValuesType.NUMERIC, term, field, value);
}
@Override
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java Sat Nov 1 09:32:06 2014
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import java.io.IOException;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Bits; // javadocs
@@ -50,7 +49,7 @@ public abstract class DocsEnum extends D
/**
* Returns term frequency in the current document, or 1 if the field was
- * indexed with {@link IndexOptions#DOCS_ONLY}. Do not call this before
+ * indexed with {@link IndexOptions#DOCS}. Do not call this before
* {@link #nextDoc} is first called, nor after {@link #nextDoc} returns
* {@link DocIdSetIterator#NO_MORE_DOCS}.
*
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Sat Nov 1 09:32:06 2014
@@ -34,90 +34,19 @@ public final class FieldInfo {
/** Internal field number */
public final int number;
- private DocValuesType docValueType;
+ private DocValuesType docValuesType = DocValuesType.NONE;
// True if any document indexed term vectors
private boolean storeTermVector;
private boolean omitNorms; // omit norms associated with indexed fields
- private IndexOptions indexOptions;
+ private IndexOptions indexOptions = IndexOptions.NONE;
private boolean storePayloads; // whether this field stores payloads together with term positions
private Map<String,String> attributes;
private long dvGen;
-
- /**
- * Controls how much information is stored in the postings lists.
- * @lucene.experimental
- */
- public static enum IndexOptions {
- // NOTE: order is important here; FieldInfo uses this
- // order to merge two conflicting IndexOptions (always
- // "downgrades" by picking the lowest).
- /**
- * Only documents are indexed: term frequencies and positions are omitted.
- * Phrase and other positional queries on the field will throw an exception, and scoring
- * will behave as if any term in the document appears only once.
- */
- // TODO: maybe rename to just DOCS?
- DOCS_ONLY,
- /**
- * Only documents and term frequencies are indexed: positions are omitted.
- * This enables normal scoring, except Phrase and other positional queries
- * will throw an exception.
- */
- DOCS_AND_FREQS,
- /**
- * Indexes documents, frequencies and positions.
- * This is a typical default for full-text search: full scoring is enabled
- * and positional queries are supported.
- */
- DOCS_AND_FREQS_AND_POSITIONS,
- /**
- * Indexes documents, frequencies, positions and offsets.
- * Character offsets are encoded alongside the positions.
- */
- DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS,
- }
-
- /**
- * DocValues types.
- * Note that DocValues is strongly typed, so a field cannot have different types
- * across different documents.
- */
- public static enum DocValuesType {
- /**
- * A per-document Number
- */
- NUMERIC,
- /**
- * A per-document byte[]. Values may be larger than
- * 32766 bytes, but different codecs may enforce their own limits.
- */
- BINARY,
- /**
- * A pre-sorted byte[]. Fields with this type only store distinct byte values
- * and store an additional offset pointer per document to dereference the shared
- * byte[]. The stored byte[] is presorted and allows access via document id,
- * ordinal and by-value. Values must be <= 32766 bytes.
- */
- SORTED,
- /**
- * A pre-sorted Number[]. Fields with this type store numeric values in sorted
- * order according to {@link Long#compare(long, long)}.
- */
- SORTED_NUMERIC,
- /**
- * A pre-sorted Set<byte[]>. Fields with this type only store distinct byte values
- * and store additional offset pointers per document to dereference the shared
- * byte[]s. The stored byte[] is presorted and allows access via document id,
- * ordinal and by-value. Values must be <= 32766 bytes.
- */
- SORTED_SET
- }
-
/**
* Sole constructor.
*
@@ -126,19 +55,24 @@ public final class FieldInfo {
public FieldInfo(String name, int number, boolean storeTermVector, boolean omitNorms,
boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues,
long dvGen, Map<String,String> attributes) {
+ if (docValues == null) {
+ throw new NullPointerException("DocValuesType cannot be null (field: \"" + name + "\")");
+ }
+ if (indexOptions == null) {
+ throw new NullPointerException("IndexOptions cannot be null (field: \"" + name + "\")");
+ }
this.name = name;
this.number = number;
- this.docValueType = docValues;
- if (indexOptions != null) {
+ this.docValuesType = docValues;
+ this.indexOptions = indexOptions;
+ if (indexOptions != IndexOptions.NONE) {
this.storeTermVector = storeTermVector;
this.storePayloads = storePayloads;
this.omitNorms = omitNorms;
- this.indexOptions = indexOptions;
} else { // for non-indexed fields, leave defaults
this.storeTermVector = false;
this.storePayloads = false;
this.omitNorms = false;
- this.indexOptions = null;
}
this.dvGen = dvGen;
this.attributes = attributes;
@@ -150,7 +84,7 @@ public final class FieldInfo {
* Always returns true (or throws IllegalStateException)
*/
public boolean checkConsistency() {
- if (indexOptions != null) {
+ if (indexOptions != IndexOptions.NONE) {
// Cannot store payloads unless positions are indexed:
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0 && storePayloads) {
throw new IllegalStateException("indexed field '" + name + "' cannot have payloads without positions");
@@ -167,7 +101,7 @@ public final class FieldInfo {
}
}
- if (dvGen != -1 && docValueType == null) {
+ if (dvGen != -1 && docValuesType == DocValuesType.NONE) {
throw new IllegalStateException("field '" + name + "' cannot have a docvalues update generation without having docvalues");
}
@@ -180,26 +114,29 @@ public final class FieldInfo {
// should only be called by FieldInfos#addOrUpdate
void update(boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
+ if (indexOptions == null) {
+ throw new NullPointerException("IndexOptions cannot be null (field: \"" + name + "\")");
+ }
//System.out.println("FI.update field=" + name + " indexed=" + indexed + " omitNorms=" + omitNorms + " this.omitNorms=" + this.omitNorms);
if (this.indexOptions != indexOptions) {
- if (this.indexOptions == null) {
+ if (this.indexOptions == IndexOptions.NONE) {
this.indexOptions = indexOptions;
- } else if (indexOptions != null) {
+ } else if (indexOptions != IndexOptions.NONE) {
// downgrade
this.indexOptions = this.indexOptions.compareTo(indexOptions) < 0 ? this.indexOptions : indexOptions;
}
}
- if (this.indexOptions != null) { // if updated field data is not for indexing, leave the updates out
+ if (this.indexOptions != IndexOptions.NONE) { // if updated field data is not for indexing, leave the updates out
this.storeTermVector |= storeTermVector; // once vector, always vector
this.storePayloads |= storePayloads;
// Awkward: only drop norms if incoming update is indexed:
- if (indexOptions != null && this.omitNorms != omitNorms) {
+ if (indexOptions != IndexOptions.NONE && this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
}
}
- if (this.indexOptions == null || this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
+ if (this.indexOptions == IndexOptions.NONE || this.indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
// cannot store payloads if we don't store positions:
this.storePayloads = false;
}
@@ -207,14 +144,17 @@ public final class FieldInfo {
}
void setDocValuesType(DocValuesType type) {
- if (docValueType != null && docValueType != type) {
- throw new IllegalArgumentException("cannot change DocValues type from " + docValueType + " to " + type + " for field \"" + name + "\"");
+ if (type == null) {
+ throw new NullPointerException("DocValuesType cannot be null (field: \"" + name + "\")");
+ }
+ if (docValuesType != DocValuesType.NONE && docValuesType != type) {
+ throw new IllegalArgumentException("cannot change DocValues type from " + docValuesType + " to " + type + " for field \"" + name + "\"");
}
- docValueType = type;
+ docValuesType = type;
assert checkConsistency();
}
- /** Returns IndexOptions for the field, or null if the field is not indexed */
+ /** Returns IndexOptions for the field, or IndexOptions.NONE if the field is not indexed */
public IndexOptions getIndexOptions() {
return indexOptions;
}
@@ -223,14 +163,15 @@ public final class FieldInfo {
* Returns true if this field has any docValues.
*/
public boolean hasDocValues() {
- return docValueType != null;
+ return docValuesType != DocValuesType.NONE;
}
/**
- * Returns {@link DocValuesType} of the docValues. this may be null if the field has no docvalues.
+ * Returns {@link DocValuesType} of the docValues; this is
+ * {@code DocValuesType.NONE} if the field has no docvalues.
*/
public DocValuesType getDocValuesType() {
- return docValueType;
+ return docValuesType;
}
/** Sets the docValues generation of this field. */
@@ -253,7 +194,7 @@ public final class FieldInfo {
}
void setStorePayloads() {
- if (indexOptions != null && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
+ if (indexOptions != IndexOptions.NONE && indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
storePayloads = true;
}
assert checkConsistency();
@@ -274,10 +215,10 @@ public final class FieldInfo {
}
/**
- * Returns true if this field is indexed (has non-null {@link #getIndexOptions}).
+ * Returns true if this field is indexed ({@link #getIndexOptions} is not IndexOptions.NONE).
*/
public boolean isIndexed() {
- return indexOptions != null;
+ return indexOptions != IndexOptions.NONE;
}
/**
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Sat Nov 1 09:32:06 2014
@@ -25,9 +25,6 @@ import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-
/**
* Collection of {@link FieldInfo}s (accessible by number or by name).
* @lucene.experimental
@@ -72,7 +69,7 @@ public class FieldInfos implements Itera
hasVectors |= info.hasVectors();
hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS_ONLY;
+ hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS;
hasOffsets |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
hasNorms |= info.hasNorms();
hasDocValues |= info.hasDocValues();
@@ -190,18 +187,17 @@ public class FieldInfos implements Itera
* is used as the field number.
*/
synchronized int addOrGet(String fieldName, int preferredFieldNumber, DocValuesType dvType) {
- if (dvType != null) {
+ if (dvType != DocValuesType.NONE) {
DocValuesType currentDVType = docValuesType.get(fieldName);
if (currentDVType == null) {
docValuesType.put(fieldName, dvType);
- } else if (currentDVType != null && currentDVType != dvType) {
+ } else if (currentDVType != DocValuesType.NONE && currentDVType != dvType) {
throw new IllegalArgumentException("cannot change DocValues type from " + currentDVType + " to " + dvType + " for field \"" + fieldName + "\"");
}
}
Integer fieldNumber = nameToNumber.get(fieldName);
if (fieldNumber == null) {
final Integer preferredBoxed = Integer.valueOf(preferredFieldNumber);
-
if (preferredFieldNumber != -1 && !numberToName.containsKey(preferredBoxed)) {
// cool - we can use this number globally
fieldNumber = preferredBoxed;
@@ -212,7 +208,7 @@ public class FieldInfos implements Itera
}
fieldNumber = lowestUnassignedFieldNumber;
}
-
+ assert fieldNumber >= 0;
numberToName.put(fieldNumber, fieldName);
nameToNumber.put(fieldName, fieldNumber);
}
@@ -228,7 +224,7 @@ public class FieldInfos implements Itera
throw new IllegalArgumentException("field name \"" + name + "\" is already mapped to field number \"" + nameToNumber.get(name) + "\", not \"" + number + "\"");
}
DocValuesType currentDVType = docValuesType.get(name);
- if (dvType != null && currentDVType != null && dvType != currentDVType) {
+ if (dvType != DocValuesType.NONE && currentDVType != null && currentDVType != DocValuesType.NONE && dvType != currentDVType) {
throw new IllegalArgumentException("cannot change DocValues type from " + currentDVType + " to " + dvType + " for field \"" + name + "\"");
}
}
@@ -298,6 +294,9 @@ public class FieldInfos implements Itera
private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber,
boolean storeTermVector,
boolean omitNorms, boolean storePayloads, IndexOptions indexOptions, DocValuesType docValues) {
+ if (docValues == null) {
+ throw new NullPointerException("DocValuesType cannot be null");
+ }
FieldInfo fi = fieldInfo(name);
if (fi == null) {
// This field wasn't yet added to this in-RAM
@@ -313,7 +312,7 @@ public class FieldInfos implements Itera
} else {
fi.update(storeTermVector, omitNorms, storePayloads, indexOptions);
- if (docValues != null) {
+ if (docValues != DocValuesType.NONE) {
// Only pay the synchronization cost if fi does not already have a DVType
boolean updateGlobal = !fi.hasDocValues();
if (updateGlobal) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java Sat Nov 1 09:32:06 2014
@@ -23,7 +23,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
import org.apache.lucene.util.AttributeSource; // javadocs
import org.apache.lucene.util.Bits;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Sat Nov 1 09:32:06 2014
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
@@ -50,7 +49,7 @@ final class FreqProxTermsWriterPerField
public FreqProxTermsWriterPerField(FieldInvertState invertState, TermsHash termsHash, FieldInfo fieldInfo, TermsHashPerField nextPerField) {
super(fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 ? 2 : 1, invertState, termsHash, nextPerField, fieldInfo);
IndexOptions indexOptions = fieldInfo.getIndexOptions();
- assert indexOptions != null;
+ assert indexOptions != IndexOptions.NONE;
hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
@@ -196,7 +195,7 @@ final class FreqProxTermsWriterPerField
@Override
ParallelPostingsArray createPostingsArray(int size) {
IndexOptions indexOptions = fieldInfo.getIndexOptions();
- assert indexOptions != null;
+ assert indexOptions != IndexOptions.NONE;
boolean hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
boolean hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
boolean hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Nov 1 09:32:06 2014
@@ -48,7 +48,6 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.DocValuesUpdate.BinaryDocValuesUpdate;
import org.apache.lucene.index.DocValuesUpdate.NumericDocValuesUpdate;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.FieldInfos.FieldNumbers;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.MergeState.CheckAbort;
@@ -1524,6 +1523,9 @@ public class IndexWriter implements Clos
final Field f = updates[i];
final DocValuesType dvType = f.fieldType().docValueType();
if (dvType == null) {
+ throw new NullPointerException("DocValuesType cannot be null (field: \"" + f.name() + "\")");
+ }
+ if (dvType == DocValuesType.NONE) {
throw new IllegalArgumentException("can only update NUMERIC or BINARY fields! field=" + f.name());
}
if (!globalFieldNumberMap.contains(f.name(), dvType)) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableField.java Sat Nov 1 09:32:06 2014
@@ -73,8 +73,8 @@ public interface IndexableField {
* the range of that encoding.
* <p>
* It is illegal to return a boost other than 1.0f for a field that is not
- * indexed ({@link IndexableFieldType#indexOptions()} is null) or omits normalization values
- * ({@link IndexableFieldType#omitNorms()} returns true).
+ * indexed ({@link IndexableFieldType#indexOptions()} is IndexOptions.NONE) or
+ * omits normalization values ({@link IndexableFieldType#omitNorms()} returns true).
*
* @see Similarity#computeNorm(FieldInvertState)
* @see DefaultSimilarity#encodeNormValue(float)
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Sat Nov 1 09:32:06 2014
@@ -18,8 +18,6 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.analysis.Analyzer; // javadocs
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
/**
* Describes the properties of a field.
@@ -38,7 +36,8 @@ public interface IndexableFieldType {
* can be accessed in a document-oriented way from
* {@link IndexReader#getTermVector(int,String)}.
* <p>
- * This option is illegal if {@link #indexOptions()} returns null.
+ * This option is illegal if {@link #indexOptions()} returns
+ * IndexOptions.NONE.
*/
public boolean storeTermVectors();
@@ -78,13 +77,12 @@ public interface IndexableFieldType {
public boolean omitNorms();
/** {@link IndexOptions}, describing what should be
- * recorded into the inverted index, or null if this field
- * is not indexed */
+ * recorded into the inverted index */
public IndexOptions indexOptions();
/**
- * DocValues {@link DocValuesType}: if non-null then the field's value
- * will be indexed into docValues.
+ * DocValues {@link DocValuesType}: how the field's value will be indexed
+ * into docValues.
*/
public DocValuesType docValueType();
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesFieldUpdates.java Sat Nov 1 09:32:06 2014
@@ -86,7 +86,7 @@ class NumericDocValuesFieldUpdates exten
private int size;
public NumericDocValuesFieldUpdates(String field, int maxDoc) {
- super(field, FieldInfo.DocValuesType.NUMERIC);
+ super(field, DocValuesType.NUMERIC);
bitsPerValue = PackedInts.bitsRequired(maxDoc - 1);
docs = new PagedMutable(1, PAGE_SIZE, bitsPerValue, PackedInts.COMPACT);
values = new PagedGrowableWriter(1, PAGE_SIZE, 1, PackedInts.FAST);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SegmentReader.java Sat Nov 1 09:32:06 2014
@@ -33,7 +33,7 @@ import org.apache.lucene.codecs.NormsPro
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.document.FieldTypes;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Accountable;
@@ -342,14 +342,14 @@ public final class SegmentReader extends
// returns the FieldInfo that corresponds to the given field and type, or
// null if the field does not exist, or not indexed as the requested
- // DovDocValuesType.
+ // DOVDOCVALUESTYPE.
private FieldInfo getDVField(String field, DocValuesType type) {
FieldInfo fi = fieldInfos.fieldInfo(field);
if (fi == null) {
// Field does not exist
return null;
}
- if (fi.getDocValuesType() == null) {
+ if (fi.getDocValuesType() == DocValuesType.NONE) {
// Field was not indexed with doc values
return null;
}
@@ -394,7 +394,7 @@ public final class SegmentReader extends
// Field does not exist
return null;
}
- if (fi.getDocValuesType() == null) {
+ if (fi.getDocValuesType() == DocValuesType.NONE) {
// Field was not indexed with doc values
return null;
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/SlowCompositeReaderWrapper.java Sat Nov 1 09:32:06 2014
@@ -23,7 +23,6 @@ import java.util.Map;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
import org.apache.lucene.index.MultiDocValues.OrdinalMap;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Sat Nov 1 09:32:06 2014
@@ -111,7 +111,7 @@ final class TermVectorsConsumerPerField
@Override
boolean start(IndexableField field, boolean first) {
- assert field.fieldType().indexOptions() != null;
+ assert field.fieldType().indexOptions() != IndexOptions.NONE;
if (first) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/package.html?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/package.html (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/index/package.html Sat Nov 1 09:32:06 2014
@@ -161,7 +161,7 @@ while ((docid = docsAndPositionsEnum.nex
of occurrences of this term across all documents. Note that this statistic
is unavailable (returns <code>-1</code>) if term frequencies were omitted
from the index
- ({@link org.apache.lucene.index.FieldInfo.IndexOptions#DOCS_ONLY DOCS_ONLY})
+ ({@link org.apache.lucene.index.IndexOptions#DOCS DOCS})
for the field. Like docFreq(), it will also count occurrences that appear in
deleted documents.
</ul>
@@ -194,7 +194,7 @@ while ((docid = docsAndPositionsEnum.nex
field, and like totalTermFreq() it will also count occurrences that appear in
deleted documents, and will be unavailable (returns <code>-1</code>) if term
frequencies were omitted from the index
- ({@link org.apache.lucene.index.FieldInfo.IndexOptions#DOCS_ONLY DOCS_ONLY})
+ ({@link org.apache.lucene.index.IndexOptions#DOCS DOCS})
for the field.
</ul>
</p>
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/CachingWrapperFilter.java Sat Nov 1 09:32:06 2014
@@ -67,18 +67,12 @@ public class CachingWrapperFilter extend
* instance is use as a placeholder in the cache instead of the <code>null</code> value.
*/
protected DocIdSet docIdSetToCache(DocIdSet docIdSet, LeafReader reader) throws IOException {
- if (docIdSet == null) {
- // this is better than returning null, as the nonnull result can be cached
- return EMPTY;
- } else if (docIdSet.isCacheable()) {
+ if (docIdSet == null || docIdSet.isCacheable()) {
return docIdSet;
} else {
final DocIdSetIterator it = docIdSet.iterator();
- // null is allowed to be returned by iterator(),
- // in this case we wrap with the sentinel set,
- // which is cacheable.
if (it == null) {
- return EMPTY;
+ return null;
} else {
return cacheImpl(it, reader);
}
@@ -106,6 +100,10 @@ public class CachingWrapperFilter extend
} else {
missCount++;
docIdSet = docIdSetToCache(filter.getDocIdSet(context, null), reader);
+ if (docIdSet == null) {
+ // We use EMPTY as a sentinel for the empty set, which is cacheable
+ docIdSet = EMPTY;
+ }
assert docIdSet.isCacheable();
cache.put(key, docIdSet);
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSet.java Sat Nov 1 09:32:06 2014
@@ -52,7 +52,12 @@ public abstract class FilteredDocIdSet e
public FilteredDocIdSet(DocIdSet innerSet) {
_innerSet = innerSet;
}
-
+
+ /** Return the wrapped {@link DocIdSet}. */
+ public DocIdSet getDelegate() {
+ return _innerSet;
+ }
+
/** This DocIdSet implementation is cacheable if the inner set is cacheable. */
@Override
public boolean isCacheable() {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/FilteredDocIdSetIterator.java Sat Nov 1 09:32:06 2014
@@ -41,6 +41,11 @@ public abstract class FilteredDocIdSetIt
doc = -1;
}
+ /** Return the wrapped {@link DocIdSetIterator}. */
+ public DocIdSetIterator getDelegate() {
+ return _innerIter;
+ }
+
/**
* Validation method to determine whether a docid should be in the result set.
* @param doc docid to be tested
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Sat Nov 1 09:32:06 2014
@@ -25,8 +25,8 @@ import org.apache.lucene.index.LeafReade
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.util.BitDocIdSet;
import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.DocIdSetBuilder;
/**
* A wrapper for {@link MultiTermQuery}, that exposes its
@@ -100,7 +100,7 @@ public class MultiTermQueryWrapperFilter
final TermsEnum termsEnum = query.getTermsEnum(terms);
assert termsEnum != null;
- DocIdSetBuilder builder = new DocIdSetBuilder(context.reader().maxDoc());
+ BitDocIdSet.Builder builder = new BitDocIdSet.Builder(context.reader().maxDoc());
DocsEnum docs = null;
while (termsEnum.next() != null) {
docs = termsEnum.docs(acceptDocs, docs, DocsEnum.FLAG_NONE);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitDocIdSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitDocIdSet.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitDocIdSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitDocIdSet.java Sat Nov 1 09:32:06 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
@@ -75,4 +77,113 @@ public class BitDocIdSet extends DocIdSe
return getClass().getSimpleName() + "(set=" + set + ",cost=" + cost + ")";
}
+ /**
+ * A builder of {@link DocIdSet}s that supports random access.
+ * @lucene.internal
+ */
+ public static final class Builder {
+
+ private final int maxDoc;
+ private final int threshold;
+ private SparseFixedBitSet sparseSet;
+ private FixedBitSet denseSet;
+
+ // we cache an upper bound of the cost of this builder so that we don't have
+ // to re-compute approximateCardinality on the sparse set every time
+ private long costUpperBound;
+
+ /** Create a new instance that can hold <code>maxDoc</code> documents and is optionally <code>full</code>. */
+ public Builder(int maxDoc, boolean full) {
+ this.maxDoc = maxDoc;
+ threshold = maxDoc >>> 10;
+ if (full) {
+ denseSet = new FixedBitSet(maxDoc);
+ denseSet.set(0, maxDoc);
+ }
+ }
+
+ /** Create a new empty instance. */
+ public Builder(int maxDoc) {
+ this(maxDoc, false);
+ }
+
+ /**
+ * Add the content of the provided {@link DocIdSetIterator} to this builder.
+ */
+ public void or(DocIdSetIterator it) throws IOException {
+ if (denseSet != null) {
+ // already upgraded
+ denseSet.or(it);
+ return;
+ }
+
+ final long itCost = it.cost();
+ costUpperBound += itCost;
+ if (costUpperBound >= threshold) {
+ costUpperBound = (sparseSet == null ? 0 : sparseSet.approximateCardinality()) + itCost;
+
+ if (costUpperBound >= threshold) {
+ // upgrade
+ denseSet = new FixedBitSet(maxDoc);
+ denseSet.or(it);
+ if (sparseSet != null) {
+ denseSet.or(new BitSetIterator(sparseSet, 0L));
+ }
+ return;
+ }
+ }
+
+ // we are still sparse
+ if (sparseSet == null) {
+ sparseSet = new SparseFixedBitSet(maxDoc);
+ }
+ sparseSet.or(it);
+ }
+
+ /**
+ * Removes from this builder documents that are not contained in <code>it</code>.
+ */
+ public void and(DocIdSetIterator it) throws IOException {
+ if (denseSet != null) {
+ denseSet.and(it);
+ } else if (sparseSet != null) {
+ sparseSet.and(it);
+ }
+ }
+
+ /**
+ * Removes from this builder documents that are contained in <code>it</code>.
+ */
+ public void andNot(DocIdSetIterator it) throws IOException {
+ if (denseSet != null) {
+ denseSet.andNot(it);
+ } else if (denseSet != null) {
+ denseSet.andNot(it);
+ }
+ }
+
+ /**
+ * Build a {@link DocIdSet} that contains all doc ids that have been added.
+ * This method may return <tt>null</tt> if no documents were addded to this
+ * builder.
+ * NOTE: this is a destructive operation, the builder should not be used
+ * anymore after this method has been called.
+ */
+ public BitDocIdSet build() {
+ final BitDocIdSet result;
+ if (denseSet != null) {
+ result = new BitDocIdSet(denseSet);
+ } else if (sparseSet != null) {
+ result = new BitDocIdSet(sparseSet);
+ } else {
+ result = null;
+ }
+ denseSet = null;
+ sparseSet = null;
+ costUpperBound = 0;
+ return result;
+ }
+
+ }
+
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitSet.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/BitSet.java Sat Nov 1 09:32:06 2014
@@ -54,10 +54,15 @@ public abstract class BitSet implements
return cardinality();
}
+ /** Returns the index of the last set bit before or on the index specified.
+ * -1 is returned if there are no more set bits.
+ */
+ public abstract int prevSetBit(int index);
+
/** Returns the index of the first set bit starting at the index specified.
* {@link DocIdSetIterator#NO_MORE_DOCS} is returned if there are no more set bits.
*/
- public abstract int nextSetBit(int i);
+ public abstract int nextSetBit(int index);
/** Assert that the current doc is -1. */
protected final void assertUnpositioned(DocIdSetIterator iter) {
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java Sat Nov 1 09:32:06 2014
@@ -203,9 +203,7 @@ public final class FixedBitSet extends B
return DocIdSetIterator.NO_MORE_DOCS;
}
- /** Returns the index of the last set bit before or on the index specified.
- * -1 is returned if there are no more set bits.
- */
+ @Override
public int prevSetBit(int index) {
assert index >= 0 && index < numBits: "index=" + index + " numBits=" + numBits;
int i = index >> 6;
Modified: lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/java/org/apache/lucene/util/SparseFixedBitSet.java Sat Nov 1 09:32:06 2014
@@ -290,59 +290,70 @@ public class SparseFixedBitSet extends B
assert i < length;
final int i4096 = i >>> 12;
final long index = indices[i4096];
+ final long[] bitArray = this.bits[i4096];
int i64 = i >>> 6;
- long indexBits = index >>> i64;
+ int o = Long.bitCount(index & ((1L << i64) - 1));
+ if ((index & (1L << i64)) != 0) {
+ // There is at least one bit that is set in the current long, check if
+ // one of them is after i
+ final long bits = bitArray[o] >>> i; // shifts are mod 64
+ if (bits != 0) {
+ return i + Long.numberOfTrailingZeros(bits);
+ }
+ o += 1;
+ }
+ final long indexBits = index >>> i64 >>> 1;
if (indexBits == 0) {
- // if the index is zero, it means that there is no value in the
- // current block, so return the first document of the next block
- // or
- // if neither the i64-th bit or any other bit on its left is set then
- // it means that there are no more documents in this block, go to the
- // next one
+ // no more bits are set in the current block of 4096 bits, go to the next one
return firstDoc(i4096 + 1);
- } else {
- // We know we still have some 64-bits blocks that have bits set, let's
- // advance to the next one by skipping trailing zeros of the index
- int i1 = i & 0x3F;
- int trailingZeros = Long.numberOfTrailingZeros(indexBits);
- if (trailingZeros != 0) {
- // no bits in the current long, go to the next one
- i64 += trailingZeros;
- i1 = 0;
- }
-
- // So now we are on a sub 64-bits block that has values
- assert (index & (1L << i64)) != 0;
- // we count the number of ones on the left of i64 to figure out the
- // index of the long that contains the bits we are interested in
- int longIndex = Long.bitCount(index & ((1L << i64) - 1)); // shifts are mod 64 in java
- final long[] longArray = bits[i4096];
- assert longArray[longIndex] != 0;
- long bits = longArray[longIndex] >>> i1; // shifts are mod 64 in java
- if (bits != 0L) {
- // hurray, we found some non-zero bits, this gives us the next document:
- i1 += Long.numberOfTrailingZeros(bits);
- return (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
- }
-
- // otherwise it means that although we were on a sub-64 block that contains
- // documents, all documents of this sub-block have already been consumed
- // so two cases:
- indexBits = index >>> i64 >>> 1; // we don't shift by (i64+1) otherwise we might shift by a multiple of 64 which is a no-op
- if (indexBits == 0) {
- // Case 1: this was the last long of the block of 4096 bits, then go
- // to the next block
- return firstDoc(i4096 + 1);
- }
- // Case 2: go to the next sub 64-bits block in the current block of 4096 bits
- // by skipping trailing zeros of the index
- trailingZeros = Long.numberOfTrailingZeros(indexBits);
- i64 += 1 + trailingZeros;
- bits = longArray[longIndex + 1];
- assert bits != 0;
- i1 = Long.numberOfTrailingZeros(bits);
- return (i4096 << 12) | ((i64 & 0x3F) << 6) | i1;
}
+ // there are still set bits
+ i64 += 1 + Long.numberOfTrailingZeros(indexBits);
+ final long bits = bitArray[o];
+ return (i64 << 6) | Long.numberOfTrailingZeros(bits);
+ }
+
+ /** Return the last document that occurs on or before the provided block index. */
+ private int lastDoc(int i4096) {
+ long index;
+ while (i4096 >= 0) {
+ index = indices[i4096];
+ if (index != 0) {
+ final int i64 = 63 - Long.numberOfLeadingZeros(index);
+ final long bits = this.bits[i4096][Long.bitCount(index) - 1];
+ return (i4096 << 12) | (i64 << 6) | (63 - Long.numberOfLeadingZeros(bits));
+ }
+ i4096 -= 1;
+ }
+ return -1;
+ }
+
+ @Override
+ public int prevSetBit(int i) {
+ assert i >= 0;
+ final int i4096 = i >>> 12;
+ final long index = indices[i4096];
+ final long[] bitArray = this.bits[i4096];
+ int i64 = i >>> 6;
+ final long indexBits = index & ((1L << i64) - 1);
+ final int o = Long.bitCount(indexBits);
+ if ((index & (1L << i64)) != 0) {
+ // There is at least one bit that is set in the same long, check if there
+ // is one bit that is set that is lower than i
+ final long bits = bitArray[o] & ((1L << i << 1) - 1);
+ if (bits != 0) {
+ return (i64 << 6) | (63 - Long.numberOfLeadingZeros(bits));
+ }
+ }
+ if (indexBits == 0) {
+ // no more bits are set in this block, go find the last bit in the
+ // previous block
+ return lastDoc(i4096 - 1);
+ }
+ // go to the previous long
+ i64 = 63 - Long.numberOfLeadingZeros(indexBits);
+ final long bits = bitArray[o - 1];
+ return (i4096 << 12) | (i64 << 6) | (63 - Long.numberOfLeadingZeros(bits));
}
/** Return the long bits at the given <code>i64</code> index. */
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Sat Nov 1 09:32:06 2014
@@ -28,8 +28,8 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.FieldTypes;
import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Terms;
@@ -312,7 +312,7 @@ public class TestMockAnalyzer extends Ba
fieldTypes.enableTermVectors("f");
fieldTypes.enableTermVectorPositions("f");
fieldTypes.enableTermVectorOffsets("f");
- fieldTypes.setIndexOptions("f", IndexOptions.DOCS_ONLY);
+ fieldTypes.setIndexOptions("f", IndexOptions.DOCS);
fieldTypes.setMultiValued("f");
doc.addAtom("f", "a");
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat2.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat2.java Sat Nov 1 09:32:06 2014
@@ -22,8 +22,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
@@ -67,7 +66,10 @@ public class TestBlockPostingsFormat2 ex
private Document newDocument() {
Document doc = new Document();
- for (IndexOptions option : FieldInfo.IndexOptions.values()) {
+ for (IndexOptions option : IndexOptions.values()) {
+ if (option == IndexOptions.NONE) {
+ continue;
+ }
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
// turn on tvs for a cross-check, since we rely upon checkindex in this test (for now)
ft.setStoreTermVectors(true);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestBlockPostingsFormat3.java Sat Nov 1 09:32:06 2014
@@ -32,18 +32,18 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.LeafReader;
-import org.apache.lucene.index.LeafReaderContext;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
@@ -90,7 +90,7 @@ public class TestBlockPostingsFormat3 ex
FieldType docsOnlyType = new FieldType(TextField.TYPE_NOT_STORED);
// turn this on for a cross-check
docsOnlyType.setStoreTermVectors(true);
- docsOnlyType.setIndexOptions(IndexOptions.DOCS_ONLY);
+ docsOnlyType.setIndexOptions(IndexOptions.DOCS);
FieldType docsAndFreqsType = new FieldType(TextField.TYPE_NOT_STORED);
// turn this on for a cross-check
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/codecs/lucene50/TestLucene50NormsFormat.java Sat Nov 1 09:32:06 2014
@@ -43,85 +43,88 @@ public class TestLucene50NormsFormat ext
public void testNormMapSimple() {
NormMap map = new NormMap();
- map.add(10);
- map.add(5);
- map.add(4);
- map.add(10);
+ map.add((byte)4);
+ map.add((byte) 10);
+ map.add((byte) 5);
+ map.add((byte)10);
assertEquals(3, map.size);
// first come, first serve ord assignment
-
- // encode
- assertEquals(0, map.getOrd(10));
- assertEquals(1, map.getOrd(5));
- assertEquals(2, map.getOrd(4));
-
- // decode
- long decode[] = map.getDecodeTable();
- assertEquals(10, decode[0]);
- assertEquals(5, decode[1]);
- assertEquals(4, decode[2]);
-
- // freqs
- int freqs[] = map.getFreqs();
- assertEquals(2, freqs[0]);
- assertEquals(1, freqs[1]);
- assertEquals(1, freqs[2]);
-
- assertEquals(2, map.maxFreq());
+ assertEquals(0, map.ord((byte) 4));
+ assertEquals(1, map.ord((byte) 10));
+ assertEquals(2, map.ord((byte) 5));
+
+ assertEquals(4, map.values[0]);
+ assertEquals(10, map.values[1]);
+ assertEquals(5, map.values[2]);
+
+ assertEquals(1, map.freqs[0]);
+ assertEquals(2, map.freqs[1]);
+ assertEquals(1, map.freqs[2]);
+
+ // optimizing reorders the ordinals
+ map.optimizeOrdinals();
+ assertEquals(0, map.ord((byte)10));
+ assertEquals(1, map.ord((byte)4));
+ assertEquals(2, map.ord((byte)5));
+
+ assertEquals(10, map.values[0]);
+ assertEquals(4, map.values[1]);
+ assertEquals(5, map.values[2]);
+
+ assertEquals(2, map.freqs[0]);
+ assertEquals(1, map.freqs[1]);
+ assertEquals(1, map.freqs[2]);
}
public void testNormMapRandom() {
- Map<Long,Integer> freqs = new HashMap<>();
- Map<Long,Integer> ords = new HashMap<>();
-
- Set<Long> uniqueValuesSet = new HashSet<>();
+
+ Set<Byte> uniqueValuesSet = new HashSet<>();
int numUniqValues = TestUtil.nextInt(random(), 1, 256);
for (int i = 0; i < numUniqValues; i++) {
- if (random().nextBoolean()) {
- uniqueValuesSet.add(TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE));
- } else {
- uniqueValuesSet.add(TestUtil.nextLong(random(), Byte.MIN_VALUE, Byte.MAX_VALUE));
- }
+ uniqueValuesSet.add(Byte.valueOf((byte)TestUtil.nextInt(random(), Byte.MIN_VALUE, Byte.MAX_VALUE)));
}
-
- Long uniqueValues[] = uniqueValuesSet.toArray(new Long[uniqueValuesSet.size()]);
-
+ Byte uniqueValues[] = uniqueValuesSet.toArray(new Byte[uniqueValuesSet.size()]);
+
+ Map<Byte,Integer> freqs = new HashMap<>();
NormMap map = new NormMap();
int numdocs = TestUtil.nextInt(random(), 1, 100000);
for (int i = 0; i < numdocs; i++) {
- long value = uniqueValues[random().nextInt(uniqueValues.length)];
+ byte value = uniqueValues[random().nextInt(uniqueValues.length)];
// now add to both expected and actual
map.add(value);
-
- Integer ord = ords.get(value);
- if (ord == null) {
- ord = ords.size();
- ords.put(value, ord);
- freqs.put(value, 1);
+ if (freqs.containsKey(value)) {
+ freqs.put(value, freqs.get(value) + 1);
} else {
- freqs.put(value, freqs.get(value)+1);
+ freqs.put(value, 1);
}
}
-
- // value -> ord
- assertEquals(ords.size(), map.size);
- for (Map.Entry<Long,Integer> kv : ords.entrySet()) {
- assertEquals(kv.getValue().intValue(), map.getOrd(kv.getKey()));
+
+ assertEquals(freqs.size(), map.size);
+ for (Map.Entry<Byte,Integer> kv : freqs.entrySet()) {
+ byte value = kv.getKey();
+ int freq = kv.getValue();
+ int ord = map.ord(value);
+ assertEquals(freq, map.freqs[ord]);
+ assertEquals(value, map.values[ord]);
}
-
- // ord -> value
- Map<Long,Integer> reversed = new HashMap<>();
- long table[] = map.getDecodeTable();
- for (int i = 0; i < map.size; i++) {
- reversed.put(table[i], i);
+
+ // optimizing should reorder ordinals from greatest to least frequency
+ map.optimizeOrdinals();
+ // recheck consistency
+ assertEquals(freqs.size(), map.size);
+ for (Map.Entry<Byte,Integer> kv : freqs.entrySet()) {
+ byte value = kv.getKey();
+ int freq = kv.getValue();
+ int ord = map.ord(value);
+ assertEquals(freq, map.freqs[ord]);
+ assertEquals(value, map.values[ord]);
}
- assertEquals(ords, reversed);
-
- // freqs
- int freqTable[] = map.getFreqs();
- for (int i = 0; i < map.size; i++) {
- assertEquals(freqs.get(table[i]).longValue(), freqTable[i]);
+ // also check descending freq
+ int prevFreq = map.freqs[0];
+ for (int i = 1; i < map.size; ++i) {
+ assertTrue(prevFreq >= map.freqs[i]);
+ prevFreq = map.freqs[i];
}
}
}
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument.java Sat Nov 1 09:32:06 2014
@@ -25,7 +25,7 @@ import java.util.List;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.RandomIndexWriter;
@@ -64,7 +64,7 @@ public class TestDocument extends Lucene
assertTrue(binaryFld.binaryValue() != null);
assertTrue(binaryFld.fieldType().stored());
- assertNull(binaryFld.fieldType().indexOptions());
+ assertEquals(IndexOptions.NONE, binaryFld.fieldType().indexOptions());
String binaryTest = doc.getBinaryValue("binary").utf8ToString();
assertTrue(binaryTest.equals(binaryVal));
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestDocument2.java Sat Nov 1 09:32:06 2014
@@ -27,8 +27,8 @@ import org.apache.lucene.codecs.blocktre
import org.apache.lucene.codecs.lucene50.Lucene50Codec;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -627,7 +627,7 @@ public class TestDocument2 extends Lucen
IndexWriterConfig iwc = newIndexWriterConfig();
IndexWriter w = new IndexWriter(dir, iwc);
FieldTypes fieldTypes = w.getFieldTypes();
- fieldTypes.setIndexOptions("id", IndexOptions.DOCS_ONLY);
+ fieldTypes.setIndexOptions("id", IndexOptions.DOCS);
Document2 doc = w.newDocument();
doc.addStored("id", new BytesRef(new byte[1]));
@@ -678,7 +678,7 @@ public class TestDocument2 extends Lucen
IndexWriterConfig iwc = newIndexWriterConfig();
IndexWriter w = new IndexWriter(dir, iwc);
FieldTypes fieldTypes = w.getFieldTypes();
- fieldTypes.setIndexOptions("id", IndexOptions.DOCS_ONLY);
+ fieldTypes.setIndexOptions("id", IndexOptions.DOCS);
fieldTypes.setTermsDictBlockSize("id", 10);
for(int i=0;i<10;i++) {
@@ -863,7 +863,7 @@ public class TestDocument2 extends Lucen
IndexWriterConfig iwc = newIndexWriterConfig();
IndexWriter w = new IndexWriter(dir, iwc);
FieldTypes fieldTypes = w.getFieldTypes();
- fieldTypes.setIndexOptions("foo", null);
+ fieldTypes.setIndexOptions("foo", IndexOptions.NONE);
Document2 doc = w.newDocument();
doc.addAtom("foo", "bar");
@@ -888,7 +888,7 @@ public class TestDocument2 extends Lucen
IndexWriterConfig iwc = newIndexWriterConfig();
IndexWriter w = new IndexWriter(dir, iwc);
FieldTypes fieldTypes = w.getFieldTypes();
- fieldTypes.setDocValuesType("foo", null);
+ fieldTypes.setDocValuesType("foo", DocValuesType.NONE);
Document2 doc = w.newDocument();
doc.addInt("foo", 17);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/document/TestFieldType.java Sat Nov 1 09:32:06 2014
@@ -18,8 +18,8 @@ package org.apache.lucene.document;
*/
import org.apache.lucene.document.FieldType.NumericType;
-import org.apache.lucene.index.FieldInfo.DocValuesType;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.DocValuesType;
+import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.util.LuceneTestCase;
/** simple testcases for concrete impl of IndexableFieldType */
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java Sat Nov 1 09:32:06 2014
@@ -24,13 +24,11 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TimeUnits;
-
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
/**
@@ -66,7 +64,7 @@ public class Test2BPostings extends Luce
Document doc = new Document();
FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
ft.setOmitNorms(true);
- ft.setIndexOptions(IndexOptions.DOCS_ONLY);
+ ft.setIndexOptions(IndexOptions.DOCS);
Field field = new Field("field", new MyTokenStream(), ft);
doc.add(field);
Modified: lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java?rev=1635908&r1=1635907&r2=1635908&view=diff
==============================================================================
--- lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java (original)
+++ lucene/dev/branches/lucene6005/lucene/core/src/test/org/apache/lucene/index/Test2BPostingsBytes.java Sat Nov 1 09:32:06 2014
@@ -26,17 +26,13 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
-import org.apache.lucene.util.TimeUnits;
import org.apache.lucene.util.LuceneTestCase.Monster;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
-import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
-
/**
* Test indexes 2B docs with 65k freqs each,
* so you get > Integer.MAX_VALUE postings data for the term